Search in sources :

Example 11 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class RemoteNamespaceQueryTest method setup.

@BeforeClass
public static void setup() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TEMPORARY_FOLDER.newFolder().getAbsolutePath());
    Injector injector = AppFabricTestHelper.getInjector(cConf);
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    appFabricServer = injector.getInstance(AppFabricServer.class);
    appFabricServer.startAndWait();
    DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    waitForService(discoveryServiceClient, Constants.Service.DATASET_MANAGER);
    waitForService(discoveryServiceClient, Constants.Service.APP_FABRIC_HTTP);
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    queryClient = injector.getInstance(RemoteNamespaceQueryClient.class);
    namespacedLocationFactory = injector.getInstance(NamespacedLocationFactory.class);
}
Also used : DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) Injector(com.google.inject.Injector) TransactionManager(org.apache.tephra.TransactionManager) AppFabricServer(co.cask.cdap.internal.app.services.AppFabricServer) NamespaceAdmin(co.cask.cdap.common.namespace.NamespaceAdmin) DatasetService(co.cask.cdap.data2.datafabric.dataset.service.DatasetService) RemoteNamespaceQueryClient(co.cask.cdap.common.namespace.RemoteNamespaceQueryClient) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) CConfiguration(co.cask.cdap.common.conf.CConfiguration) BeforeClass(org.junit.BeforeClass)

Example 12 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class DatasetServiceTestBase method initializeAndStartService.

protected static void initializeAndStartService(CConfiguration cConf) throws Exception {
    // TODO: this whole method is a mess. Streamline it!
    injector = Guice.createInjector(new ConfigModule(cConf), new DiscoveryRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule().getModule(), new NamespaceClientRuntimeModule().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new TransactionInMemoryModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class).in(Singleton.class);
            install(new FactoryModuleBuilder().implement(DatasetDefinitionRegistry.class, DefaultDatasetDefinitionRegistry.class).build(DatasetDefinitionRegistryFactory.class));
            // through the injector, we only need RemoteDatasetFramework in these tests
            bind(RemoteDatasetFramework.class);
            bind(OwnerStore.class).to(InMemoryOwnerStore.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    });
    AuthorizationEnforcer authEnforcer = injector.getInstance(AuthorizationEnforcer.class);
    AuthenticationContext authenticationContext = injector.getInstance(AuthenticationContext.class);
    DiscoveryService discoveryService = injector.getInstance(DiscoveryService.class);
    discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    dsFramework = injector.getInstance(RemoteDatasetFramework.class);
    // Tx Manager to support working with datasets
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    TransactionSystemClient txSystemClient = injector.getInstance(TransactionSystemClient.class);
    TransactionSystemClientService txSystemClientService = new DelegatingTransactionSystemClientService(txSystemClient);
    NamespacedLocationFactory namespacedLocationFactory = injector.getInstance(NamespacedLocationFactory.class);
    SystemDatasetInstantiatorFactory datasetInstantiatorFactory = new SystemDatasetInstantiatorFactory(locationFactory, dsFramework, cConf);
    // ok to pass null, since the impersonator won't actually be called, if kerberos security is not enabled
    Impersonator impersonator = new DefaultImpersonator(cConf, null);
    DatasetAdminService datasetAdminService = new DatasetAdminService(dsFramework, cConf, locationFactory, datasetInstantiatorFactory, new NoOpMetadataStore(), impersonator);
    ImmutableSet<HttpHandler> handlers = ImmutableSet.<HttpHandler>of(new DatasetAdminOpHTTPHandler(datasetAdminService));
    MetricsCollectionService metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    opExecutorService = new DatasetOpExecutorService(cConf, discoveryService, metricsCollectionService, handlers);
    opExecutorService.startAndWait();
    Map<String, DatasetModule> defaultModules = injector.getInstance(Key.get(new TypeLiteral<Map<String, DatasetModule>>() {
    }, Constants.Dataset.Manager.DefaultDatasetModules.class));
    ImmutableMap<String, DatasetModule> modules = ImmutableMap.<String, DatasetModule>builder().putAll(defaultModules).putAll(DatasetMetaTableUtil.getModules()).build();
    registryFactory = injector.getInstance(DatasetDefinitionRegistryFactory.class);
    inMemoryDatasetFramework = new InMemoryDatasetFramework(registryFactory, modules);
    DiscoveryExploreClient exploreClient = new DiscoveryExploreClient(discoveryServiceClient, authenticationContext);
    ExploreFacade exploreFacade = new ExploreFacade(exploreClient, cConf);
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    namespaceAdmin.create(NamespaceMeta.DEFAULT);
    ownerAdmin = injector.getInstance(OwnerAdmin.class);
    NamespaceQueryAdmin namespaceQueryAdmin = injector.getInstance(NamespaceQueryAdmin.class);
    TransactionExecutorFactory txExecutorFactory = new DynamicTransactionExecutorFactory(txSystemClient);
    DatasetTypeManager typeManager = new DatasetTypeManager(cConf, locationFactory, txSystemClientService, txExecutorFactory, inMemoryDatasetFramework, impersonator);
    DatasetOpExecutor opExecutor = new InMemoryDatasetOpExecutor(dsFramework);
    DatasetInstanceManager instanceManager = new DatasetInstanceManager(txSystemClientService, txExecutorFactory, inMemoryDatasetFramework);
    PrivilegesManager privilegesManager = injector.getInstance(PrivilegesManager.class);
    DatasetTypeService typeService = new DatasetTypeService(typeManager, namespaceAdmin, namespacedLocationFactory, authEnforcer, privilegesManager, authenticationContext, cConf, impersonator, txSystemClientService, inMemoryDatasetFramework, txExecutorFactory, defaultModules);
    instanceService = new DatasetInstanceService(typeService, instanceManager, opExecutor, exploreFacade, namespaceQueryAdmin, ownerAdmin, authEnforcer, privilegesManager, authenticationContext);
    service = new DatasetService(cConf, discoveryService, discoveryServiceClient, metricsCollectionService, opExecutor, new HashSet<DatasetMetricsReporter>(), typeService, instanceService);
    // Start dataset service, wait for it to be discoverable
    service.startAndWait();
    waitForService(Constants.Service.DATASET_EXECUTOR);
    waitForService(Constants.Service.DATASET_MANAGER);
    // this usually happens while creating a namespace, however not doing that in data fabric tests
    Locations.mkdirsIfNotExists(namespacedLocationFactory.get(NamespaceId.DEFAULT));
}
Also used : RemoteDatasetFramework(co.cask.cdap.data2.datafabric.dataset.RemoteDatasetFramework) InMemoryDatasetOpExecutor(co.cask.cdap.data2.datafabric.dataset.service.executor.InMemoryDatasetOpExecutor) AuthenticationContext(co.cask.cdap.security.spi.authentication.AuthenticationContext) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) DatasetAdminOpHTTPHandler(co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetAdminOpHTTPHandler) TransactionInMemoryModule(org.apache.tephra.runtime.TransactionInMemoryModule) AuthorizationEnforcer(co.cask.cdap.security.spi.authorization.AuthorizationEnforcer) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) NoOpMetricsCollectionService(co.cask.cdap.common.metrics.NoOpMetricsCollectionService) ExploreFacade(co.cask.cdap.explore.client.ExploreFacade) DynamicTransactionExecutorFactory(co.cask.cdap.data.runtime.DynamicTransactionExecutorFactory) TransactionExecutorFactory(co.cask.cdap.data2.transaction.TransactionExecutorFactory) NoOpMetadataStore(co.cask.cdap.data2.metadata.store.NoOpMetadataStore) DatasetDefinitionRegistryFactory(co.cask.cdap.data2.dataset2.DatasetDefinitionRegistryFactory) SystemDatasetRuntimeModule(co.cask.cdap.data.runtime.SystemDatasetRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) InMemoryDatasetFramework(co.cask.cdap.data2.dataset2.InMemoryDatasetFramework) HashSet(java.util.HashSet) HttpHandler(co.cask.http.HttpHandler) DatasetInstanceManager(co.cask.cdap.data2.datafabric.dataset.instance.DatasetInstanceManager) MetricsCollectionService(co.cask.cdap.api.metrics.MetricsCollectionService) NoOpMetricsCollectionService(co.cask.cdap.common.metrics.NoOpMetricsCollectionService) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) OwnerAdmin(co.cask.cdap.security.impersonation.OwnerAdmin) NamespaceAdmin(co.cask.cdap.common.namespace.NamespaceAdmin) PrivilegesManager(co.cask.cdap.security.spi.authorization.PrivilegesManager) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) DatasetTypeManager(co.cask.cdap.data2.datafabric.dataset.type.DatasetTypeManager) TransactionManager(org.apache.tephra.TransactionManager) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) DelegatingTransactionSystemClientService(co.cask.cdap.data2.transaction.DelegatingTransactionSystemClientService) NamespaceClientRuntimeModule(co.cask.cdap.common.namespace.guice.NamespaceClientRuntimeModule) ConfigModule(co.cask.cdap.common.guice.ConfigModule) FactoryModuleBuilder(com.google.inject.assistedinject.FactoryModuleBuilder) DynamicTransactionExecutorFactory(co.cask.cdap.data.runtime.DynamicTransactionExecutorFactory) DatasetModule(co.cask.cdap.api.dataset.module.DatasetModule) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) DiscoveryExploreClient(co.cask.cdap.explore.client.DiscoveryExploreClient) SystemDatasetInstantiatorFactory(co.cask.cdap.data.dataset.SystemDatasetInstantiatorFactory) TypeLiteral(com.google.inject.TypeLiteral) NamespaceQueryAdmin(co.cask.cdap.common.namespace.NamespaceQueryAdmin) TransactionSystemClientService(co.cask.cdap.data2.transaction.TransactionSystemClientService) DelegatingTransactionSystemClientService(co.cask.cdap.data2.transaction.DelegatingTransactionSystemClientService) DiscoveryService(org.apache.twill.discovery.DiscoveryService) DatasetAdminService(co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetAdminService) DatasetOpExecutor(co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutor) InMemoryDatasetOpExecutor(co.cask.cdap.data2.datafabric.dataset.service.executor.InMemoryDatasetOpExecutor) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) Impersonator(co.cask.cdap.security.impersonation.Impersonator) DefaultImpersonator(co.cask.cdap.security.impersonation.DefaultImpersonator) DefaultImpersonator(co.cask.cdap.security.impersonation.DefaultImpersonator) InMemoryOwnerStore(co.cask.cdap.security.impersonation.InMemoryOwnerStore) OwnerStore(co.cask.cdap.security.impersonation.OwnerStore) AbstractModule(com.google.inject.AbstractModule) DatasetOpExecutorService(co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService)

Example 13 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class DatasetOpExecutorServiceTest method setUp.

@Before
public void setUp() throws Exception {
    Configuration hConf = new Configuration();
    CConfiguration cConf = CConfiguration.create();
    File datasetDir = new File(TMP_FOLDER.newFolder(), "datasetUser");
    Assert.assertTrue(datasetDir.mkdirs());
    cConf.set(Constants.Dataset.Manager.OUTPUT_DIR, datasetDir.getAbsolutePath());
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, "localhost");
    cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost");
    cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule().getModule(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new TransactionMetricsModule(), new ExploreClientModule(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    });
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    managerService = injector.getInstance(DatasetService.class);
    managerService.startAndWait();
    dsFramework = injector.getInstance(DatasetFramework.class);
    // find host
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    endpointStrategy = new RandomEndpointStrategy(discoveryClient.discover(Constants.Service.DATASET_MANAGER));
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    namespaceAdmin.create(NamespaceMeta.DEFAULT);
    namespaceAdmin.create(new NamespaceMeta.Builder().setName(bob.getParent()).build());
}
Also used : IOModule(co.cask.cdap.common.guice.IOModule) NamespaceClientRuntimeModule(co.cask.cdap.common.namespace.guice.NamespaceClientRuntimeModule) DataSetServiceModules(co.cask.cdap.data.runtime.DataSetServiceModules) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) ConfigModule(co.cask.cdap.common.guice.ConfigModule) DatasetService(co.cask.cdap.data2.datafabric.dataset.service.DatasetService) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) Injector(com.google.inject.Injector) KafkaClientModule(co.cask.cdap.common.guice.KafkaClientModule) NamespaceMeta(co.cask.cdap.proto.NamespaceMeta) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) NamespaceAdmin(co.cask.cdap.common.namespace.NamespaceAdmin) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) CConfiguration(co.cask.cdap.common.conf.CConfiguration) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) AbstractModule(com.google.inject.AbstractModule) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) TransactionManager(org.apache.tephra.TransactionManager) File(java.io.File) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) RandomEndpointStrategy(co.cask.cdap.common.discovery.RandomEndpointStrategy) Before(org.junit.Before)

Example 14 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class LocalLogAppenderResilientTest method testResilientLogging.

@Test
public void testResilientLogging() throws Exception {
    Configuration hConf = new Configuration();
    CConfiguration cConf = CConfiguration.create();
    File datasetDir = new File(tmpFolder.newFolder(), "datasetUser");
    //noinspection ResultOfMethodCallIgnored
    datasetDir.mkdirs();
    cConf.set(Constants.Dataset.Manager.OUTPUT_DIR, datasetDir.getAbsolutePath());
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, "localhost");
    cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost");
    cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule().getModule(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new TransactionMetricsModule(), new ExploreClientModule(), new LoggingModules().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
        }
    });
    TransactionManager txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    DatasetOpExecutorService opExecutorService = injector.getInstance(DatasetOpExecutorService.class);
    opExecutorService.startAndWait();
    // Start the logging before starting the service.
    LoggingContextAccessor.setLoggingContext(new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN", "INSTANCE"));
    String logBaseDir = "trl-log/log_files_" + new Random(System.currentTimeMillis()).nextLong();
    cConf.set(LoggingConfiguration.LOG_BASE_DIR, logBaseDir);
    cConf.setInt(LoggingConfiguration.LOG_MAX_FILE_SIZE_BYTES, 20 * 1024);
    final LogAppender appender = injector.getInstance(LocalLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestResilientLogging");
    int failureMsgCount = 3;
    final CountDownLatch failureLatch = new CountDownLatch(failureMsgCount);
    LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
    loggerContext.getStatusManager().add(new StatusListener() {

        @Override
        public void addStatusEvent(Status status) {
            if (status.getLevel() != Status.ERROR || status.getOrigin() != appender) {
                return;
            }
            Throwable cause = status.getThrowable();
            if (cause != null) {
                Throwable rootCause = Throwables.getRootCause(cause);
                if (rootCause instanceof ServiceUnavailableException) {
                    String serviceName = ((ServiceUnavailableException) rootCause).getServiceName();
                    if (Constants.Service.DATASET_MANAGER.equals(serviceName)) {
                        failureLatch.countDown();
                    }
                }
            }
        }
    });
    Logger logger = LoggerFactory.getLogger("TestResilientLogging");
    for (int i = 0; i < failureMsgCount; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    // Wait for the three failure to append to happen
    // The wait time has to be > 3 seconds because DatasetServiceClient has 1 second timeout on discovery
    failureLatch.await(5, TimeUnit.SECONDS);
    // Start dataset service, wait for it to be discoverable
    DatasetService dsService = injector.getInstance(DatasetService.class);
    dsService.startAndWait();
    final CountDownLatch startLatch = new CountDownLatch(1);
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    discoveryClient.discover(Constants.Service.DATASET_MANAGER).watchChanges(new ServiceDiscovered.ChangeListener() {

        @Override
        public void onChange(ServiceDiscovered serviceDiscovered) {
            if (!Iterables.isEmpty(serviceDiscovered)) {
                startLatch.countDown();
            }
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    startLatch.await(5, TimeUnit.SECONDS);
    // Do some more logging after the service is started.
    for (int i = 5; i < 10; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    appender.stop();
    // Verify - we should have at least 5 events.
    LoggingContext loggingContext = new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "", "RUN", "INSTANCE");
    FileLogReader logTail = injector.getInstance(FileLogReader.class);
    LoggingTester.LogCallback logCallback1 = new LoggingTester.LogCallback();
    logTail.getLogPrev(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> allEvents = logCallback1.getEvents();
    Assert.assertTrue(allEvents.toString(), allEvents.size() >= 5);
    // Finally - stop all services
    Services.chainStop(dsService, opExecutorService, txManager);
}
Also used : NoOpOwnerAdmin(co.cask.cdap.security.impersonation.NoOpOwnerAdmin) DataSetServiceModules(co.cask.cdap.data.runtime.DataSetServiceModules) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) LoggingConfiguration(co.cask.cdap.logging.LoggingConfiguration) LocalLogAppender(co.cask.cdap.logging.framework.local.LocalLogAppender) DatasetService(co.cask.cdap.data2.datafabric.dataset.service.DatasetService) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) FileLogReader(co.cask.cdap.logging.read.FileLogReader) Random(java.util.Random) Injector(com.google.inject.Injector) KafkaClientModule(co.cask.cdap.common.guice.KafkaClientModule) FlowletLoggingContext(co.cask.cdap.logging.context.FlowletLoggingContext) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) Status(ch.qos.logback.core.status.Status) LogEvent(co.cask.cdap.logging.read.LogEvent) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) ServiceDiscovered(org.apache.twill.discovery.ServiceDiscovered) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) CConfiguration(co.cask.cdap.common.conf.CConfiguration) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) TransactionManager(org.apache.tephra.TransactionManager) File(java.io.File) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) IOModule(co.cask.cdap.common.guice.IOModule) NamespaceClientRuntimeModule(co.cask.cdap.common.namespace.guice.NamespaceClientRuntimeModule) LoggingContext(co.cask.cdap.common.logging.LoggingContext) FlowletLoggingContext(co.cask.cdap.logging.context.FlowletLoggingContext) ConfigModule(co.cask.cdap.common.guice.ConfigModule) ServiceUnavailableException(co.cask.cdap.common.ServiceUnavailableException) Logger(org.slf4j.Logger) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) CountDownLatch(java.util.concurrent.CountDownLatch) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) LoggerContext(ch.qos.logback.classic.LoggerContext) ServiceUnavailableException(co.cask.cdap.common.ServiceUnavailableException) LoggingModules(co.cask.cdap.logging.guice.LoggingModules) AbstractModule(com.google.inject.AbstractModule) DatasetOpExecutorService(co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService) StatusListener(ch.qos.logback.core.status.StatusListener) Test(org.junit.Test)

Example 15 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class MetricsSuiteTestBase method startMetricsService.

public static Injector startMetricsService(CConfiguration conf) {
    Injector injector = Guice.createInjector(Modules.override(new ConfigModule(conf), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new MetricsHandlerModule(), new MetricsClientRuntimeModule().getInMemoryModules(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new ExploreClientModule(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(LogReader.class).to(MockLogReader.class).in(Scopes.SINGLETON);
            bind(Store.class).to(DefaultStore.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
        }
    }));
    transactionManager = injector.getInstance(TransactionManager.class);
    transactionManager.startAndWait();
    dsOpService = injector.getInstance(DatasetOpExecutor.class);
    dsOpService.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    metrics = injector.getInstance(MetricsQueryService.class);
    metrics.startAndWait();
    collectionService = injector.getInstance(MetricsCollectionService.class);
    collectionService.startAndWait();
    logReader = injector.getInstance(LogReader.class);
    // initialize the dataset instantiator
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    EndpointStrategy metricsEndPoints = new RandomEndpointStrategy(discoveryClient.discover(Constants.Service.METRICS));
    Discoverable discoverable = metricsEndPoints.pick(1L, TimeUnit.SECONDS);
    Assert.assertNotNull("Could not discover metrics service", discoverable);
    port = discoverable.getSocketAddress().getPort();
    return injector;
}
Also used : NamespaceClientRuntimeModule(co.cask.cdap.common.namespace.guice.NamespaceClientRuntimeModule) DataSetServiceModules(co.cask.cdap.data.runtime.DataSetServiceModules) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) ConfigModule(co.cask.cdap.common.guice.ConfigModule) UGIProvider(co.cask.cdap.security.impersonation.UGIProvider) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) MetricStore(co.cask.cdap.api.metrics.MetricStore) DefaultStore(co.cask.cdap.internal.app.store.DefaultStore) Store(co.cask.cdap.app.store.Store) DatasetService(co.cask.cdap.data2.datafabric.dataset.service.DatasetService) MetricsClientRuntimeModule(co.cask.cdap.metrics.guice.MetricsClientRuntimeModule) MockLogReader(co.cask.cdap.gateway.handlers.log.MockLogReader) EndpointStrategy(co.cask.cdap.common.discovery.EndpointStrategy) RandomEndpointStrategy(co.cask.cdap.common.discovery.RandomEndpointStrategy) Injector(com.google.inject.Injector) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) Discoverable(org.apache.twill.discovery.Discoverable) MetricsQueryService(co.cask.cdap.metrics.query.MetricsQueryService) MetricsCollectionService(co.cask.cdap.api.metrics.MetricsCollectionService) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) OwnerAdmin(co.cask.cdap.security.impersonation.OwnerAdmin) NoOpOwnerAdmin(co.cask.cdap.security.impersonation.NoOpOwnerAdmin) DatasetOpExecutor(co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutor) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) AbstractModule(com.google.inject.AbstractModule) MetricsHandlerModule(co.cask.cdap.metrics.guice.MetricsHandlerModule) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) TransactionManager(org.apache.tephra.TransactionManager) MockLogReader(co.cask.cdap.gateway.handlers.log.MockLogReader) LogReader(co.cask.cdap.logging.read.LogReader) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) RandomEndpointStrategy(co.cask.cdap.common.discovery.RandomEndpointStrategy)

Aggregations

DiscoveryServiceClient (org.apache.twill.discovery.DiscoveryServiceClient)21 Injector (com.google.inject.Injector)13 RandomEndpointStrategy (co.cask.cdap.common.discovery.RandomEndpointStrategy)12 AbstractModule (com.google.inject.AbstractModule)10 CConfiguration (co.cask.cdap.common.conf.CConfiguration)9 DiscoveryRuntimeModule (co.cask.cdap.common.guice.DiscoveryRuntimeModule)9 TransactionManager (org.apache.tephra.TransactionManager)9 EndpointStrategy (co.cask.cdap.common.discovery.EndpointStrategy)8 DatasetService (co.cask.cdap.data2.datafabric.dataset.service.DatasetService)8 ConfigModule (co.cask.cdap.common.guice.ConfigModule)7 MetricsCollectionService (co.cask.cdap.api.metrics.MetricsCollectionService)6 AuthorizationEnforcementModule (co.cask.cdap.security.authorization.AuthorizationEnforcementModule)6 Discoverable (org.apache.twill.discovery.Discoverable)6 BeforeClass (org.junit.BeforeClass)6 Test (org.junit.Test)6 AuthenticationContextModules (co.cask.cdap.security.auth.context.AuthenticationContextModules)5 AuthorizationTestModule (co.cask.cdap.security.authorization.AuthorizationTestModule)5 TransactionSystemClient (org.apache.tephra.TransactionSystemClient)5 SConfiguration (co.cask.cdap.common.conf.SConfiguration)4 IOModule (co.cask.cdap.common.guice.IOModule)4