Examples with DataSetsModules io.cdap.cdap.data.runtime.DataSetsModules used on opensource projects

Example 6 with DataSetsModules

use of io.cdap.cdap.data.runtime.DataSetsModules in project cdap by caskdata.

the class LogHttpHandlerTest method setup.

@BeforeClass
public static void setup() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    cConf.set(Constants.LogQuery.ADDRESS, InetAddress.getLoopbackAddress().getHostAddress());
    Injector injector = Guice.createInjector(Modules.override(new ConfigModule(cConf), RemoteAuthenticatorModules.getNoOpModule(), new NonCustomLocationUnitTestModule(), new InMemoryDiscoveryModule(), new LogQueryRuntimeModule().getInMemoryModules(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new ExploreClientModule(), new NamespaceAdminTestModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
            bind(LogReader.class).to(MockLogReader.class).in(Scopes.SINGLETON);
            bind(Store.class).to(DefaultStore.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
            // TODO (CDAP-14677): find a better way to inject metadata publisher
            bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
        }
    }));
    transactionManager = injector.getInstance(TransactionManager.class);
    transactionManager.startAndWait();
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    dsOpService = injector.getInstance(DatasetOpExecutorService.class);
    dsOpService.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    logQueryService = injector.getInstance(LogQueryService.class);
    logQueryService.startAndWait();
    mockLogReader = (MockLogReader) injector.getInstance(LogReader.class);
    mockLogReader.generateLogs();
    discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
}

Example 7 with DataSetsModules

use of io.cdap.cdap.data.runtime.DataSetsModules in project cdap by caskdata.

the class JobQueueDebugger method createInjector.

private static Injector createInjector() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) {
        System.out.println(String.format("Disabling authorization for %s.", JobQueueDebugger.class.getSimpleName()));
        cConf.setBoolean(Constants.Security.Authorization.ENABLED, false);
    }
    // Note: login has to happen before any objects that need Kerberos credentials are instantiated.
    SecurityUtil.loginForMasterService(cConf);
    return Guice.createInjector(new ConfigModule(cConf, HBaseConfiguration.create()), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), new ZKClientModule(), new ZKDiscoveryModule(), new DFSLocationModule(), new TwillModule(), new ExploreClientModule(), new DataFabricModules().getDistributedModules(), new DataSetsModules().getDistributedModules(), new AppFabricServiceRuntimeModule(cConf).getDistributedModules(), new ProgramRunnerRuntimeModule().getDistributedModules(), new SystemDatasetRuntimeModule().getDistributedModules(), new KafkaLogAppenderModule(), new MetricsClientRuntimeModule().getDistributedModules(), new MetricsStoreModule(), new KafkaClientModule(), CoreSecurityRuntimeModule.getDistributedModule(cConf), new AuthenticationContextModules().getMasterModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), new SecureStoreServerModule(), new MessagingClientModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(Store.class).annotatedWith(Names.named("defaultStore")).to(DefaultStore.class).in(Singleton.class);
            // This is needed because the LocalApplicationManager
            // expects a dsframework injection named datasetMDS
            bind(DatasetFramework.class).annotatedWith(Names.named("datasetMDS")).to(DatasetFramework.class).in(Singleton.class);
            // TODO (CDAP-14677): find a better way to inject metadata publisher
            bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
        }
    });
}

Example 8 with DataSetsModules

use of io.cdap.cdap.data.runtime.DataSetsModules in project cdap by caskdata.

the class UpgradeTool method createInjector.

@VisibleForTesting
Injector createInjector() {
    return Guice.createInjector(new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new DFSLocationModule(), new ZKClientModule(), new ZKDiscoveryModule(), new MessagingClientModule(), Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(DatasetFramework.class).to(InMemoryDatasetFramework.class).in(Scopes.SINGLETON);
            // the DataSetsModules().getDistributedModules() binds to RemoteDatasetFramework so override that to
            // the same InMemoryDatasetFramework
            bind(DatasetFramework.class).annotatedWith(Names.named(DataSetsModules.BASE_DATASET_FRAMEWORK)).to(DatasetFramework.class);
            bind(DatasetDefinitionRegistryFactory.class).to(DefaultDatasetDefinitionRegistryFactory.class).in(Scopes.SINGLETON);
            // CDAP-5954 Upgrade tool does not need to record lineage and metadata changes for now.
            bind(LineageWriter.class).to(NoOpLineageWriter.class);
            bind(FieldLineageWriter.class).to(NoOpLineageWriter.class);
        }
    }), new TwillModule(), new ExploreClientModule(), new ProgramRunnerRuntimeModule().getDistributedModules(), new SystemDatasetRuntimeModule().getDistributedModules(), new KafkaClientModule(), new IOModule(), CoreSecurityRuntimeModule.getDistributedModule(cConf), new AuthenticationContextModules().getMasterModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), new SecureStoreServerModule(), new DataFabricModules(UpgradeTool.class.getName()).getDistributedModules(), new AppFabricServiceRuntimeModule(cConf).getDistributedModules(), new KafkaLogAppenderModule(), // the DataFabricDistributedModule needs MetricsCollectionService binding
    new AbstractModule() {

        @Override
        protected void configure() {
            // Since Upgrade tool does not do anything with Metrics we just bind it to no-op implementations
            bind(MetricsCollectionService.class).toInstance(new NoOpMetricsCollectionService());
            bind(MetricsSystemClient.class).toInstance(new NoOpMetricsSystemClient());
        }

        @Provides
        @Singleton
        @Named("datasetInstanceManager")
        @SuppressWarnings("unused")
        public DatasetInstanceManager getDatasetInstanceManager(TransactionRunner transactionRunner) {
            return new DatasetInstanceManager(transactionRunner);
        }

        // This is needed because the LocalApplicationManager
        // expects a dsframework injection named datasetMDS
        @Provides
        @Singleton
        @Named("datasetMDS")
        @SuppressWarnings("unused")
        public DatasetFramework getInDsFramework(DatasetFramework dsFramework) {
            return dsFramework;
        }
    }, new AbstractModule() {

        @Override
        protected void configure() {
            // TODO (CDAP-14677): find a better way to inject metadata publisher
            bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
        }
    });
}

Example 9 with DataSetsModules

use of io.cdap.cdap.data.runtime.DataSetsModules in project cdap by caskdata.

the class TransactionServiceTest method testHA.

@Test(timeout = 30000)
public void testHA() throws Exception {
    // NOTE: we play with blocking/nonblocking a lot below
    // as until we integrate with "leader election" stuff, service blocks on start if it is not a leader
    // TODO: fix this by integration with generic leader election stuff
    CConfiguration cConf = CConfiguration.create();
    // tests should use the current user for HDFS
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), RemoteAuthenticatorModules.getNoOpModule(), new ZKClientModule(), new ZKDiscoveryModule(), new NonCustomLocationUnitTestModule(), new TransactionMetricsModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }, new DataFabricModules().getDistributedModules(), Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetadataStorage.class).to(NoopMetadataStorage.class);
        }
    }), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule());
    ZKClientService zkClient = injector.getInstance(ZKClientService.class);
    zkClient.startAndWait();
    try {
        final Table table = createTable("myTable");
        // tx service client
        // NOTE: we can init it earlier than we start services, it should pick them up when they are available
        TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
        TransactionExecutor txExecutor = new DefaultTransactionExecutor(txClient, ImmutableList.of((TransactionAware) table));
        // starting tx service, tx client can pick it up
        TransactionService first = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder());
        first.startAndWait();
        Assert.assertNotNull(txClient.startShort());
        verifyGetAndPut(table, txExecutor, null, "val1");
        // starting another tx service should not hurt
        TransactionService second = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder());
        // NOTE: we don't have to wait for start as client should pick it up anyways, but we do wait to ensure
        // the case with two active is handled well
        second.startAndWait();
        // wait for affect a bit
        TimeUnit.SECONDS.sleep(1);
        Assert.assertNotNull(txClient.startShort());
        verifyGetAndPut(table, txExecutor, "val1", "val2");
        // shutting down the first one is fine: we have another one to pick up the leader role
        first.stopAndWait();
        Assert.assertNotNull(txClient.startShort());
        verifyGetAndPut(table, txExecutor, "val2", "val3");
        // doing same trick again to failover to the third one
        TransactionService third = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder());
        // NOTE: we don't have to wait for start as client should pick it up anyways
        third.start();
        // stopping second one
        second.stopAndWait();
        Assert.assertNotNull(txClient.startShort());
        verifyGetAndPut(table, txExecutor, "val3", "val4");
        // releasing resources
        third.stop();
    } finally {
        try {
            dropTable("myTable");
        } finally {
            zkClient.stopAndWait();
        }
    }
}

Example 10 with DataSetsModules

use of io.cdap.cdap.data.runtime.DataSetsModules in project cdap by caskdata.

the class TransactionServiceTest method createTxService.

static TransactionService createTxService(String zkConnectionString, int txServicePort, Configuration hConf, final File outPath, @Nullable CConfiguration cConfig) {
    final CConfiguration cConf = cConfig == null ? CConfiguration.create() : cConfig;
    // tests should use the current user for HDFS
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    cConf.set(Constants.Zookeeper.QUORUM, zkConnectionString);
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, outPath.getAbsolutePath());
    cConf.set(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Integer.toString(txServicePort));
    // we want persisting for this test
    cConf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, true);
    cConf.setBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE, false);
    final Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new NonCustomLocationUnitTestModule(), new ZKClientModule(), new ZKDiscoveryModule(), new TransactionMetricsModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }, new DataFabricModules().getDistributedModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new DataSetsModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule());
    injector.getInstance(ZKClientService.class).startAndWait();
    return injector.getInstance(TransactionService.class);
}