Examples with KafkaClientModule io.cdap.cdap.common.guice.KafkaClientModule used on opensource projects

Example 1 with KafkaClientModule

use of io.cdap.cdap.common.guice.KafkaClientModule in project cdap by caskdata.

the class PreviewRunnerTwillRunnable method createInjector.

@VisibleForTesting
static Injector createInjector(CConfiguration cConf, Configuration hConf, PreviewRequestPollerInfo pollerInfo) {
    List<Module> modules = new ArrayList<>();
    byte[] pollerInfoBytes = Bytes.toBytes(new Gson().toJson(pollerInfo));
    SConfiguration sConf = SConfiguration.create();
    modules.add(new ConfigModule(cConf, hConf, sConf));
    modules.add(RemoteAuthenticatorModules.getDefaultModule());
    modules.add(new PreviewConfigModule(cConf, hConf, sConf));
    modules.add(new IOModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    // If MasterEnvironment is not available, assuming it is the old hadoop stack with ZK, Kafka
    MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
    if (masterEnv == null) {
        modules.add(new ZKClientModule());
        modules.add(new ZKDiscoveryModule());
        modules.add(new KafkaClientModule());
        modules.add(new KafkaLogAppenderModule());
    } else {
        modules.add(new AbstractModule() {

            @Override
            protected void configure() {
                bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
                bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
            }
        });
        modules.add(new RemoteLogAppenderModule());
    }
    modules.add(new PreviewRunnerManagerModule().getDistributedModules());
    modules.add(new DataSetServiceModules().getStandaloneModules());
    modules.add(new DataSetsModules().getStandaloneModules());
    modules.add(new AppFabricServiceRuntimeModule(cConf).getStandaloneModules());
    modules.add(new ProgramRunnerRuntimeModule().getStandaloneModules());
    modules.add(new MetricsStoreModule());
    modules.add(new MessagingClientModule());
    modules.add(new AuditModule());
    modules.add(new SecureStoreClientModule());
    modules.add(new MetadataReaderWriterModules().getStandaloneModules());
    modules.add(new DFSLocationModule());
    modules.add(new MetadataServiceModule());
    modules.add(new CoreSecurityRuntimeModule().getInMemoryModules());
    modules.add(new AuthenticationContextModules().getMasterWorkerModule());
    modules.add(new AuthorizationModule());
    modules.add(new AuthorizationEnforcementModule().getNoOpModules());
    modules.add(Modules.override(new DataFabricModules("master").getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            // Bind transaction system to a constant one, basically no transaction, with every write become
            // visible immediately.
            // TODO: Ideally we shouldn't need this at all. However, it is needed now to satisfy dependencies
            bind(TransactionSystemClientService.class).to(DelegatingTransactionSystemClientService.class);
            bind(TransactionSystemClient.class).to(ConstantTransactionSystemClient.class);
            bind(ExploreClient.class).to(UnsupportedExploreClient.class);
            bind(PreviewRequestPollerInfoProvider.class).toInstance(() -> pollerInfoBytes);
        }
    }));
    return Guice.createInjector(modules);
}

Example 2 with KafkaClientModule

use of io.cdap.cdap.common.guice.KafkaClientModule in project cdap by caskdata.

the class DistributedProgramContainerModule method addOnPremiseModules.

private void addOnPremiseModules(List<Module> modules) {
    CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf);
    modules.add(new AuthenticationContextModules().getMasterModule());
    modules.add(coreSecurityModule);
    // If MasterEnvironment is not available, assuming it is the old hadoop stack with ZK, Kafka
    MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
    if (masterEnv == null) {
        modules.add(new ZKClientModule());
        modules.add(new ZKDiscoveryModule());
        modules.add(new KafkaClientModule());
        modules.add(new KafkaLogAppenderModule());
        return;
    }
    if (coreSecurityModule.requiresZKClient()) {
        modules.add(new ZKClientModule());
    }
    modules.add(new AbstractModule() {

        @Override
        protected void configure() {
            bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
            bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    });
    modules.add(new RemoteLogAppenderModule());
}

Example 3 with KafkaClientModule

use of io.cdap.cdap.common.guice.KafkaClientModule in project cdap by caskdata.

the class TaskWorkerTwillRunnable method createInjector.

@VisibleForTesting
static Injector createInjector(CConfiguration cConf, Configuration hConf) {
    List<Module> modules = new ArrayList<>();
    CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf);
    modules.add(new ConfigModule(cConf, hConf));
    modules.add(RemoteAuthenticatorModules.getDefaultModule());
    modules.add(new LocalLocationModule());
    modules.add(new IOModule());
    modules.add(new AuthenticationContextModules().getMasterWorkerModule());
    modules.add(coreSecurityModule);
    modules.add(new MessagingClientModule());
    modules.add(new SystemAppModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    // If MasterEnvironment is not available, assuming it is the old hadoop stack with ZK, Kafka
    MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
    if (masterEnv == null) {
        modules.add(new ZKClientModule());
        modules.add(new ZKDiscoveryModule());
        modules.add(new KafkaClientModule());
        modules.add(new KafkaLogAppenderModule());
    } else {
        modules.add(new AbstractModule() {

            @Override
            protected void configure() {
                bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
                bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
            }
        });
        modules.add(new RemoteLogAppenderModule());
        if (coreSecurityModule.requiresZKClient()) {
            modules.add(new ZKClientModule());
        }
    }
    return Guice.createInjector(modules);
}

Example 4 with KafkaClientModule

use of io.cdap.cdap.common.guice.KafkaClientModule in project cdap by caskdata.

the class DatasetOpExecutorServiceTest method setUp.

@Before
public void setUp() throws Exception {
    Configuration hConf = new Configuration();
    CConfiguration cConf = CConfiguration.create();
    File datasetDir = new File(TMP_FOLDER.newFolder(), "datasetUser");
    Assert.assertTrue(datasetDir.mkdirs());
    cConf.set(Constants.Dataset.Manager.OUTPUT_DIR, datasetDir.getAbsolutePath());
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, "localhost");
    cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost");
    cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getNoOpModule(), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new InMemoryDiscoveryModule(), new NonCustomLocationUnitTestModule(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new TransactionMetricsModule(), new ExploreClientModule(), new NamespaceAdminTestModule(), new AuthenticationContextModules().getMasterModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
            bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
        }
    });
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    dsOpExecService = injector.getInstance(DatasetOpExecutorService.class);
    dsOpExecService.startAndWait();
    managerService = injector.getInstance(DatasetService.class);
    managerService.startAndWait();
    dsFramework = injector.getInstance(DatasetFramework.class);
    // find host
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    endpointStrategy = new RandomEndpointStrategy(() -> discoveryClient.discover(Constants.Service.DATASET_MANAGER));
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    namespaceAdmin.create(NamespaceMeta.DEFAULT);
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(bob.getParent()).build());
}

Example 5 with KafkaClientModule

use of io.cdap.cdap.common.guice.KafkaClientModule in project cdap by caskdata.

the class TwillAppLifecycleEventHandler method initialize.

@Override
public void initialize(EventHandlerContext context) {
    super.initialize(context);
    this.runningPublished = new AtomicBoolean();
    this.twillRunId = context.getRunId();
    Map<String, String> configs = context.getSpecification().getConfigs();
    this.programRunId = GSON.fromJson(configs.get("programRunId"), ProgramRunId.class);
    this.clusterMode = ClusterMode.valueOf(configs.get("clusterMode"));
    this.runtimeMonitorType = RuntimeMonitorType.valueOf(configs.get("monitorType"));
    // Fetch cConf and hConf from resources jar
    File cConfFile = new File("resources.jar/resources/" + CDAP_CONF_FILE_NAME);
    File hConfFile = new File("resources.jar/resources/" + HADOOP_CONF_FILE_NAME);
    if (!cConfFile.exists()) {
        // This shouldn't happen, unless CDAP is misconfigured
        throw new IllegalArgumentException("Missing cConf file " + cConfFile.getAbsolutePath());
    }
    try {
        // Load the configuration from the XML files serialized from the cdap master.
        CConfiguration cConf = CConfiguration.create();
        cConf.clear();
        cConf.addResource(cConfFile.toURI().toURL());
        Configuration hConf = new Configuration();
        if (hConfFile.exists()) {
            hConf.clear();
            hConf.addResource(hConfFile.toURI().toURL());
        }
        // Create the injector to create a program state writer
        List<Module> modules = new ArrayList<>(Arrays.asList(new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getDefaultModule(), new MessagingClientModule(), new AbstractModule() {

            @Override
            protected void configure() {
                bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class);
            }
        }));
        switch(clusterMode) {
            case ON_PREMISE:
                modules.add(new AuthenticationContextModules().getProgramContainerModule(cConf));
                modules.add(new ZKClientModule());
                modules.add(new ZKDiscoveryModule());
                modules.add(new KafkaClientModule());
                break;
            case ISOLATED:
                modules.add(new AuthenticationContextModules().getProgramContainerModule(cConf));
                modules.add(new RemoteExecutionDiscoveryModule());
                modules.add(new AbstractModule() {

                    @Override
                    protected void configure() {
                        bind(RuntimeMonitorType.class).toInstance(runtimeMonitorType);
                    }
                });
                break;
        }
        Injector injector = Guice.createInjector(modules);
        if (clusterMode == ClusterMode.ON_PREMISE) {
            zkClientService = injector.getInstance(ZKClientService.class);
            zkClientService.startAndWait();
        }
        ProgramStateWriter programStateWriter = injector.getInstance(ProgramStateWriter.class);
        MessagingService messagingService = injector.getInstance(MessagingService.class);
        programStateWriterWithHeartBeat = new ProgramStateWriterWithHeartBeat(programRunId, programStateWriter, messagingService, cConf);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}