Examples with DFSLocationModule io.cdap.cdap.common.guice.DFSLocationModule used on opensource projects

Example 1 with DFSLocationModule

use of io.cdap.cdap.common.guice.DFSLocationModule in project cdap by caskdata.

the class RemoteExecutionJobMain method initialize.

@VisibleForTesting
RemoteExecutionRuntimeJobEnvironment initialize(CConfiguration cConf) throws Exception {
    zkServer = InMemoryZKServer.builder().build();
    zkServer.startAndWait();
    InetSocketAddress zkAddr = ResolvingDiscoverable.resolve(zkServer.getLocalAddress());
    String zkConnectStr = String.format("%s:%d", zkAddr.getHostString(), zkAddr.getPort());
    LOG.debug("In memory ZK started at {}", zkConnectStr);
    cConf.set(Constants.Zookeeper.QUORUM, zkConnectStr);
    Injector injector = Guice.createInjector(new ConfigModule(cConf), RemoteAuthenticatorModules.getDefaultModule(), new DFSLocationModule(), new InMemoryDiscoveryModule(), new TwillModule(), new AuthenticationContextModules().getProgramContainerModule(cConf), new AbstractModule() {

        @Override
        protected void configure() {
            // don't need to perform any impersonation from within user programs
            bind(UGIProvider.class).to(CurrentUGIProvider.class).in(Scopes.SINGLETON);
            // Binds a no-op SecureStore for the TwillModule to setup TokenSecureStoreRenewer.
            bind(SecureStore.class).toInstance(new SecureStore() {

                @Override
                public List<SecureStoreMetadata> list(String namespace) {
                    return Collections.emptyList();
                }

                @Override
                public SecureStoreData get(String namespace, String name) throws Exception {
                    throw new NotFoundException("Secure key " + name + " not found in namespace " + namespace);
                }
            });
        }
    });
    Map<String, String> properties = new HashMap<>();
    properties.put(Constants.Zookeeper.QUORUM, zkConnectStr);
    locationFactory = injector.getInstance(LocationFactory.class);
    locationFactory.create("/").mkdirs();
    twillRunnerService = injector.getInstance(TwillRunnerService.class);
    twillRunnerService.start();
    if (UserGroupInformation.isSecurityEnabled()) {
        TokenSecureStoreRenewer secureStoreRenewer = injector.getInstance(TokenSecureStoreRenewer.class);
        secureStoreUpdateCancellable = twillRunnerService.setSecureStoreRenewer(secureStoreRenewer, 30000L, secureStoreRenewer.getUpdateInterval(), 30000L, TimeUnit.MILLISECONDS);
    }
    return new RemoteExecutionRuntimeJobEnvironment(locationFactory, twillRunnerService, properties);
}

Example 2 with DFSLocationModule

use of io.cdap.cdap.common.guice.DFSLocationModule in project cdap by caskdata.

the class PreviewRunnerTwillRunnable method createInjector.

@VisibleForTesting
static Injector createInjector(CConfiguration cConf, Configuration hConf, PreviewRequestPollerInfo pollerInfo) {
    List<Module> modules = new ArrayList<>();
    byte[] pollerInfoBytes = Bytes.toBytes(new Gson().toJson(pollerInfo));
    SConfiguration sConf = SConfiguration.create();
    modules.add(new ConfigModule(cConf, hConf, sConf));
    modules.add(RemoteAuthenticatorModules.getDefaultModule());
    modules.add(new PreviewConfigModule(cConf, hConf, sConf));
    modules.add(new IOModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    // If MasterEnvironment is not available, assuming it is the old hadoop stack with ZK, Kafka
    MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
    if (masterEnv == null) {
        modules.add(new ZKClientModule());
        modules.add(new ZKDiscoveryModule());
        modules.add(new KafkaClientModule());
        modules.add(new KafkaLogAppenderModule());
    } else {
        modules.add(new AbstractModule() {

            @Override
            protected void configure() {
                bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
                bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
            }
        });
        modules.add(new RemoteLogAppenderModule());
    }
    modules.add(new PreviewRunnerManagerModule().getDistributedModules());
    modules.add(new DataSetServiceModules().getStandaloneModules());
    modules.add(new DataSetsModules().getStandaloneModules());
    modules.add(new AppFabricServiceRuntimeModule(cConf).getStandaloneModules());
    modules.add(new ProgramRunnerRuntimeModule().getStandaloneModules());
    modules.add(new MetricsStoreModule());
    modules.add(new MessagingClientModule());
    modules.add(new AuditModule());
    modules.add(new SecureStoreClientModule());
    modules.add(new MetadataReaderWriterModules().getStandaloneModules());
    modules.add(new DFSLocationModule());
    modules.add(new MetadataServiceModule());
    modules.add(new CoreSecurityRuntimeModule().getInMemoryModules());
    modules.add(new AuthenticationContextModules().getMasterWorkerModule());
    modules.add(new AuthorizationModule());
    modules.add(new AuthorizationEnforcementModule().getNoOpModules());
    modules.add(Modules.override(new DataFabricModules("master").getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            // Bind transaction system to a constant one, basically no transaction, with every write become
            // visible immediately.
            // TODO: Ideally we shouldn't need this at all. However, it is needed now to satisfy dependencies
            bind(TransactionSystemClientService.class).to(DelegatingTransactionSystemClientService.class);
            bind(TransactionSystemClient.class).to(ConstantTransactionSystemClient.class);
            bind(ExploreClient.class).to(UnsupportedExploreClient.class);
            bind(PreviewRequestPollerInfoProvider.class).toInstance(() -> pollerInfoBytes);
        }
    }));
    return Guice.createInjector(modules);
}

Example 3 with DFSLocationModule

use of io.cdap.cdap.common.guice.DFSLocationModule in project cdap by caskdata.

the class DistributedProgramContainerModule method getCoreModules.

private List<Module> getCoreModules() {
    Arguments systemArgs = programOpts.getArguments();
    ClusterMode clusterMode = systemArgs.hasOption(ProgramOptionConstants.CLUSTER_MODE) ? ClusterMode.valueOf(systemArgs.getOption(ProgramOptionConstants.CLUSTER_MODE)) : ClusterMode.ON_PREMISE;
    List<Module> modules = new ArrayList<>();
    modules.add(new ConfigModule(cConf, hConf));
    modules.add(new IOModule());
    modules.add(new DFSLocationModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    modules.add(new MessagingClientModule());
    modules.add(new AuditModule());
    modules.add(new AuthorizationEnforcementModule().getDistributedModules());
    modules.add(new SecureStoreClientModule());
    modules.add(new MetadataReaderWriterModules().getDistributedModules());
    modules.add(new NamespaceQueryAdminModule());
    modules.add(new DataSetsModules().getDistributedModules());
    modules.add(new AbstractModule() {

        @Override
        protected void configure() {
            bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class);
            bind(WorkflowStateWriter.class).to(MessagingWorkflowStateWriter.class);
            // don't need to perform any impersonation from within user programs
            bind(UGIProvider.class).to(CurrentUGIProvider.class).in(Scopes.SINGLETON);
            // Bind ProgramId to the passed in instance programId so that we can retrieve it back later when needed.
            // For example see ProgramDiscoveryExploreClient.
            // Also binding to instance is fine here as the programId is guaranteed to not change throughout the
            // lifecycle of this program runnable
            bind(ProgramId.class).toInstance(programRunId.getParent());
            bind(ProgramRunId.class).toInstance(programRunId);
            if (serviceAnnouncer != null) {
                bind(ServiceAnnouncer.class).toInstance(serviceAnnouncer);
            }
            bind(PreferencesFetcher.class).to(RemotePreferencesFetcherInternal.class).in(Scopes.SINGLETON);
        }
    });
    addDataFabricModules(modules);
    switch(clusterMode) {
        case ON_PREMISE:
            addOnPremiseModules(modules);
            break;
        case ISOLATED:
            addIsolatedModules(modules);
            break;
        default:
    }
    return modules;
}

Example 4 with DFSLocationModule

use of io.cdap.cdap.common.guice.DFSLocationModule in project cdap by caskdata.

the class LeaderElectionMessagingServiceTest method createInjector.

private Injector createInjector(int instanceId) {
    CConfiguration cConf = CConfiguration.copy(LeaderElectionMessagingServiceTest.cConf);
    cConf.setInt(Constants.MessagingSystem.CONTAINER_INSTANCE_ID, instanceId);
    return Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new ZKDiscoveryModule(), new DFSLocationModule(), new AbstractModule() {

        @Override
        protected void configure() {
            // Bindings to services for testing only
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
            // Use the same in memory client across all injectors.
            bind(NamespaceQueryAdmin.class).toInstance(namespaceQueryAdmin);
        }
    }, new PrivateModule() {

        @Override
        protected void configure() {
            // This is very similar to bindings in distributed mode, except we bind to level db instead of HBase
            // Also the level DB has to be one instance since unit-test runs in the same process.
            bind(TableFactory.class).annotatedWith(Names.named(CachingTableFactory.DELEGATE_TABLE_FACTORY)).toInstance(levelDBTableFactory);
            // The cache must be in singleton scope
            bind(MessageTableCacheProvider.class).to(DefaultMessageTableCacheProvider.class).in(Scopes.SINGLETON);
            bind(TableFactory.class).to(CachingTableFactory.class);
            // Bind http handlers
            MessagingServerRuntimeModule.bindHandlers(binder(), Constants.MessagingSystem.HANDLER_BINDING_NAME);
            bind(MessagingService.class).to(LeaderElectionMessagingService.class).in(Scopes.SINGLETON);
            expose(MessagingService.class);
        }
    });
}

Example 5 with DFSLocationModule

use of io.cdap.cdap.common.guice.DFSLocationModule in project cdap by caskdata.

the class JobQueueDebugger method createInjector.

private static Injector createInjector() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) {
        System.out.println(String.format("Disabling authorization for %s.", JobQueueDebugger.class.getSimpleName()));
        cConf.setBoolean(Constants.Security.Authorization.ENABLED, false);
    }
    // Note: login has to happen before any objects that need Kerberos credentials are instantiated.
    SecurityUtil.loginForMasterService(cConf);
    return Guice.createInjector(new ConfigModule(cConf, HBaseConfiguration.create()), RemoteAuthenticatorModules.getDefaultModule(), new IOModule(), new ZKClientModule(), new ZKDiscoveryModule(), new DFSLocationModule(), new TwillModule(), new ExploreClientModule(), new DataFabricModules().getDistributedModules(), new DataSetsModules().getDistributedModules(), new AppFabricServiceRuntimeModule(cConf).getDistributedModules(), new ProgramRunnerRuntimeModule().getDistributedModules(), new SystemDatasetRuntimeModule().getDistributedModules(), new KafkaLogAppenderModule(), new MetricsClientRuntimeModule().getDistributedModules(), new MetricsStoreModule(), new KafkaClientModule(), CoreSecurityRuntimeModule.getDistributedModule(cConf), new AuthenticationContextModules().getMasterModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getMasterModule(), new SecureStoreServerModule(), new MessagingClientModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(Store.class).annotatedWith(Names.named("defaultStore")).to(DefaultStore.class).in(Singleton.class);
            // This is needed because the LocalApplicationManager
            // expects a dsframework injection named datasetMDS
            bind(DatasetFramework.class).annotatedWith(Names.named("datasetMDS")).to(DatasetFramework.class).in(Singleton.class);
            // TODO (CDAP-14677): find a better way to inject metadata publisher
            bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
        }
    });
}