Search in sources :

Example 11 with ZKClientModule

use of co.cask.cdap.common.guice.ZKClientModule in project cdap by caskdata.

the class LocalLogAppenderResilientTest method testResilientLogging.

@Test
public void testResilientLogging() throws Exception {
    Configuration hConf = new Configuration();
    CConfiguration cConf = CConfiguration.create();
    File datasetDir = new File(tmpFolder.newFolder(), "datasetUser");
    // noinspection ResultOfMethodCallIgnored
    datasetDir.mkdirs();
    cConf.set(Constants.Dataset.Manager.OUTPUT_DIR, datasetDir.getAbsolutePath());
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, "localhost");
    cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost");
    cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new NonCustomLocationUnitTestModule().getModule(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new TransactionMetricsModule(), new ExploreClientModule(), new LoggingModules().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
        }
    });
    TransactionManager txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    DatasetOpExecutorService opExecutorService = injector.getInstance(DatasetOpExecutorService.class);
    opExecutorService.startAndWait();
    // Start the logging before starting the service.
    LoggingContextAccessor.setLoggingContext(new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN", "INSTANCE"));
    String logBaseDir = "trl-log/log_files_" + new Random(System.currentTimeMillis()).nextLong();
    cConf.set(LoggingConfiguration.LOG_BASE_DIR, logBaseDir);
    cConf.setInt(LoggingConfiguration.LOG_MAX_FILE_SIZE_BYTES, 20 * 1024);
    final LogAppender appender = injector.getInstance(LocalLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestResilientLogging");
    int failureMsgCount = 3;
    final CountDownLatch failureLatch = new CountDownLatch(failureMsgCount);
    LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
    loggerContext.getStatusManager().add(new StatusListener() {

        @Override
        public void addStatusEvent(Status status) {
            if (status.getLevel() != Status.ERROR || status.getOrigin() != appender) {
                return;
            }
            Throwable cause = status.getThrowable();
            if (cause != null) {
                Throwable rootCause = Throwables.getRootCause(cause);
                if (rootCause instanceof ServiceUnavailableException) {
                    String serviceName = ((ServiceUnavailableException) rootCause).getServiceName();
                    if (Constants.Service.DATASET_MANAGER.equals(serviceName)) {
                        failureLatch.countDown();
                    }
                }
            }
        }
    });
    Logger logger = LoggerFactory.getLogger("TestResilientLogging");
    for (int i = 0; i < failureMsgCount; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    // Wait for the three failure to append to happen
    // The wait time has to be > 3 seconds because DatasetServiceClient has 1 second timeout on discovery
    failureLatch.await(5, TimeUnit.SECONDS);
    // Start dataset service, wait for it to be discoverable
    DatasetService dsService = injector.getInstance(DatasetService.class);
    dsService.startAndWait();
    final CountDownLatch startLatch = new CountDownLatch(1);
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    discoveryClient.discover(Constants.Service.DATASET_MANAGER).watchChanges(new ServiceDiscovered.ChangeListener() {

        @Override
        public void onChange(ServiceDiscovered serviceDiscovered) {
            if (!Iterables.isEmpty(serviceDiscovered)) {
                startLatch.countDown();
            }
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    startLatch.await(5, TimeUnit.SECONDS);
    // Do some more logging after the service is started.
    for (int i = 5; i < 10; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    appender.stop();
    // Verify - we should have at least 5 events.
    LoggingContext loggingContext = new FlowletLoggingContext("TRL_ACCT_1", "APP_1", "FLOW_1", "", "RUN", "INSTANCE");
    FileLogReader logTail = injector.getInstance(FileLogReader.class);
    LoggingTester.LogCallback logCallback1 = new LoggingTester.LogCallback();
    logTail.getLogPrev(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> allEvents = logCallback1.getEvents();
    Assert.assertTrue(allEvents.toString(), allEvents.size() >= 5);
    // Finally - stop all services
    Services.chainStop(dsService, opExecutorService, txManager);
}
Also used : NoOpOwnerAdmin(co.cask.cdap.security.impersonation.NoOpOwnerAdmin) DataSetServiceModules(co.cask.cdap.data.runtime.DataSetServiceModules) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) LoggingConfiguration(co.cask.cdap.logging.LoggingConfiguration) LocalLogAppender(co.cask.cdap.logging.framework.local.LocalLogAppender) DatasetService(co.cask.cdap.data2.datafabric.dataset.service.DatasetService) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) FileLogReader(co.cask.cdap.logging.read.FileLogReader) Random(java.util.Random) Injector(com.google.inject.Injector) KafkaClientModule(co.cask.cdap.common.guice.KafkaClientModule) FlowletLoggingContext(co.cask.cdap.logging.context.FlowletLoggingContext) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) Status(ch.qos.logback.core.status.Status) LogEvent(co.cask.cdap.logging.read.LogEvent) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) ServiceDiscovered(org.apache.twill.discovery.ServiceDiscovered) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) CConfiguration(co.cask.cdap.common.conf.CConfiguration) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) TransactionManager(org.apache.tephra.TransactionManager) File(java.io.File) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) IOModule(co.cask.cdap.common.guice.IOModule) NamespaceClientRuntimeModule(co.cask.cdap.common.namespace.guice.NamespaceClientRuntimeModule) LoggingContext(co.cask.cdap.common.logging.LoggingContext) FlowletLoggingContext(co.cask.cdap.logging.context.FlowletLoggingContext) ConfigModule(co.cask.cdap.common.guice.ConfigModule) ServiceUnavailableException(co.cask.cdap.common.ServiceUnavailableException) Logger(org.slf4j.Logger) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) CountDownLatch(java.util.concurrent.CountDownLatch) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) LoggerContext(ch.qos.logback.classic.LoggerContext) ServiceUnavailableException(co.cask.cdap.common.ServiceUnavailableException) LoggingModules(co.cask.cdap.logging.guice.LoggingModules) AbstractModule(com.google.inject.AbstractModule) DatasetOpExecutorService(co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService) StatusListener(ch.qos.logback.core.status.StatusListener) Test(org.junit.Test)

Example 12 with ZKClientModule

use of co.cask.cdap.common.guice.ZKClientModule in project cdap by caskdata.

the class TransactionServiceTest method createTxService.

static TransactionService createTxService(String zkConnectionString, int txServicePort, Configuration hConf, final File outPath, @Nullable CConfiguration cConfig) {
    final CConfiguration cConf = cConfig == null ? CConfiguration.create() : cConfig;
    // tests should use the current user for HDFS
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    cConf.set(Constants.Zookeeper.QUORUM, zkConnectionString);
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, outPath.getAbsolutePath());
    cConf.set(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Integer.toString(txServicePort));
    // we want persisting for this test
    cConf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, true);
    cConf.setBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE, false);
    final Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new NonCustomLocationUnitTestModule().getModule(), new ZKClientModule(), new DiscoveryRuntimeModule().getDistributedModules(), new TransactionMetricsModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }, new DataFabricModules().getDistributedModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new DataSetsModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule());
    injector.getInstance(ZKClientService.class).startAndWait();
    return injector.getInstance(TransactionService.class);
}
Also used : UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) ConfigModule(co.cask.cdap.common.guice.ConfigModule) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) CConfiguration(co.cask.cdap.common.conf.CConfiguration) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) AbstractModule(com.google.inject.AbstractModule) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) ZKClientService(org.apache.twill.zookeeper.ZKClientService) Injector(com.google.inject.Injector) SimpleNamespaceQueryAdmin(co.cask.cdap.common.namespace.SimpleNamespaceQueryAdmin) SystemDatasetRuntimeModule(co.cask.cdap.data.runtime.SystemDatasetRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule)

Example 13 with ZKClientModule

use of co.cask.cdap.common.guice.ZKClientModule in project cdap by caskdata.

the class ZKRouteStoreTest method init.

@BeforeClass
public static void init() throws IOException {
    zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build();
    zkServer.startAndWait();
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule());
    zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
}
Also used : ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) ZKClientService(org.apache.twill.zookeeper.ZKClientService) Injector(com.google.inject.Injector) ConfigModule(co.cask.cdap.common.guice.ConfigModule) CConfiguration(co.cask.cdap.common.conf.CConfiguration) BeforeClass(org.junit.BeforeClass)

Example 14 with ZKClientModule

use of co.cask.cdap.common.guice.ZKClientModule in project cdap by caskdata.

the class DistributedProgramRunnableModule method getCoreModules.

private List<Module> getCoreModules(final ProgramId programId, String txClientId) {
    return new ArrayList<>(Arrays.<Module>asList(new ConfigModule(cConf, hConf), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new MetricsClientRuntimeModule().getDistributedModules(), new MessagingClientModule(), new LocationRuntimeModule().getDistributedModules(), new LoggingModules().getDistributedModules(), new DiscoveryRuntimeModule().getDistributedModules(), new DataFabricModules(txClientId).getDistributedModules(), new DataSetsModules().getDistributedModules(), new ViewAdminModules().getDistributedModules(), new StreamAdminModules().getDistributedModules(), new NotificationFeedClientModule(), new AuditModule().getDistributedModules(), new NamespaceClientRuntimeModule().getDistributedModules(), new AuthorizationEnforcementModule().getDistributedModules(), new SecureStoreModules().getDistributedModules(), new AbstractModule() {

        @Override
        protected void configure() {
            // For Binding queue stuff
            bind(QueueReaderFactory.class).in(Scopes.SINGLETON);
            // For binding DataSet transaction stuff
            install(new DataFabricFacadeModule());
            bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class);
            bind(RuntimeStore.class).to(RemoteRuntimeStore.class);
            // For binding StreamWriter
            install(createStreamFactoryModule());
            // don't need to perform any impersonation from within user programs
            bind(UGIProvider.class).to(CurrentUGIProvider.class).in(Scopes.SINGLETON);
            // bind PrivilegesManager to a remote implementation, so it does not need to instantiate the authorizer
            bind(PrivilegesManager.class).to(RemotePrivilegesManager.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
            // Bind ProgramId to the passed in instance programId so that we can retrieve it back later when needed.
            // For example see ProgramDiscoveryExploreClient.
            // Also binding to instance is fine here as the programId is guaranteed to not change throughout the
            // lifecycle of this program runnable
            bind(ProgramId.class).toInstance(programId);
            // bind explore client to ProgramDiscoveryExploreClient which is aware of the programId
            bind(ExploreClient.class).to(ProgramDiscoveryExploreClient.class).in(Scopes.SINGLETON);
            // Bind the ArtifactManager implementation
            install(new FactoryModuleBuilder().implement(ArtifactManager.class, RemoteArtifactManager.class).build(ArtifactManagerFactory.class));
            // Bind the PluginFinder implementation
            bind(PluginFinder.class).to(RemotePluginFinder.class);
        }
    }));
}
Also used : IOModule(co.cask.cdap.common.guice.IOModule) MessagingClientModule(co.cask.cdap.messaging.guice.MessagingClientModule) NamespaceClientRuntimeModule(co.cask.cdap.common.namespace.guice.NamespaceClientRuntimeModule) ProgramDiscoveryExploreClient(co.cask.cdap.explore.client.ProgramDiscoveryExploreClient) ConfigModule(co.cask.cdap.common.guice.ConfigModule) FactoryModuleBuilder(com.google.inject.assistedinject.FactoryModuleBuilder) ArtifactManagerFactory(co.cask.cdap.internal.app.runtime.artifact.ArtifactManagerFactory) ArrayList(java.util.ArrayList) RuntimeStore(co.cask.cdap.app.store.RuntimeStore) RemoteRuntimeStore(co.cask.cdap.internal.app.store.remote.RemoteRuntimeStore) MetricsClientRuntimeModule(co.cask.cdap.metrics.guice.MetricsClientRuntimeModule) ViewAdminModules(co.cask.cdap.data.view.ViewAdminModules) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) MessagingProgramStateWriter(co.cask.cdap.internal.app.program.MessagingProgramStateWriter) ProgramStateWriter(co.cask.cdap.app.runtime.ProgramStateWriter) KafkaClientModule(co.cask.cdap.common.guice.KafkaClientModule) RemotePluginFinder(co.cask.cdap.internal.app.runtime.artifact.RemotePluginFinder) PluginFinder(co.cask.cdap.internal.app.runtime.artifact.PluginFinder) CurrentUGIProvider(co.cask.cdap.security.impersonation.CurrentUGIProvider) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) SecureStoreModules(co.cask.cdap.security.guice.SecureStoreModules) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) OwnerAdmin(co.cask.cdap.security.impersonation.OwnerAdmin) LocationRuntimeModule(co.cask.cdap.common.guice.LocationRuntimeModule) PrivilegesManager(co.cask.cdap.security.spi.authorization.PrivilegesManager) RemotePrivilegesManager(co.cask.cdap.security.authorization.RemotePrivilegesManager) ProgramId(co.cask.cdap.proto.id.ProgramId) LoggingModules(co.cask.cdap.logging.guice.LoggingModules) AbstractModule(com.google.inject.AbstractModule) StreamAdminModules(co.cask.cdap.data.stream.StreamAdminModules) NotificationFeedClientModule(co.cask.cdap.notifications.feeds.client.NotificationFeedClientModule) QueueReaderFactory(co.cask.cdap.internal.app.queue.QueueReaderFactory) AuditModule(co.cask.cdap.data2.audit.AuditModule) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule)

Example 15 with ZKClientModule

use of co.cask.cdap.common.guice.ZKClientModule in project cdap by caskdata.

the class TwillAppLifecycleEventHandler method initialize.

@Override
public void initialize(EventHandlerContext context) {
    super.initialize(context);
    this.runningPublished = new AtomicBoolean();
    this.twillRunId = context.getRunId();
    this.programRunId = GSON.fromJson(context.getSpecification().getConfigs().get("programRunId"), ProgramRunId.class);
    // Fetch cConf and hConf from resources jar
    File cConfFile = new File("resources.jar/resources/" + CDAP_CONF_FILE_NAME);
    File hConfFile = new File("resources.jar/resources/" + HADOOP_CONF_FILE_NAME);
    if (cConfFile.exists() && hConfFile.exists()) {
        CConfiguration cConf = CConfiguration.create();
        cConf.clear();
        Configuration hConf = new Configuration();
        hConf.clear();
        try {
            cConf.addResource(cConfFile.toURI().toURL());
            hConf.addResource(hConfFile.toURI().toURL());
            // Create the injector to inject a program state writer
            Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new KafkaClientModule(), new DiscoveryRuntimeModule().getDistributedModules(), new MessagingClientModule(), new AbstractModule() {

                @Override
                protected void configure() {
                    bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class);
                }
            });
            zkClientService = injector.getInstance(ZKClientService.class);
            zkClientService.startAndWait();
            this.programStateWriter = injector.getInstance(ProgramStateWriter.class);
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    } else {
        LOG.warn("{} and {} were not found in the resources.jar. Not recording program states", CDAP_CONF_FILE_NAME, HADOOP_CONF_FILE_NAME);
        this.programStateWriter = new NoOpProgramStateWriter();
    }
}
Also used : MessagingClientModule(co.cask.cdap.messaging.guice.MessagingClientModule) NoOpProgramStateWriter(co.cask.cdap.app.runtime.NoOpProgramStateWriter) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Configuration(org.apache.hadoop.conf.Configuration) ConfigModule(co.cask.cdap.common.guice.ConfigModule) MessagingProgramStateWriter(co.cask.cdap.internal.app.program.MessagingProgramStateWriter) CConfiguration(co.cask.cdap.common.conf.CConfiguration) AbstractModule(com.google.inject.AbstractModule) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) ZKClientService(org.apache.twill.zookeeper.ZKClientService) MessagingProgramStateWriter(co.cask.cdap.internal.app.program.MessagingProgramStateWriter) ProgramStateWriter(co.cask.cdap.app.runtime.ProgramStateWriter) NoOpProgramStateWriter(co.cask.cdap.app.runtime.NoOpProgramStateWriter) Injector(com.google.inject.Injector) KafkaClientModule(co.cask.cdap.common.guice.KafkaClientModule) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) File(java.io.File) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule)

Aggregations

ConfigModule (co.cask.cdap.common.guice.ConfigModule)27 ZKClientModule (co.cask.cdap.common.guice.ZKClientModule)27 DiscoveryRuntimeModule (co.cask.cdap.common.guice.DiscoveryRuntimeModule)25 AbstractModule (com.google.inject.AbstractModule)21 DataSetsModules (co.cask.cdap.data.runtime.DataSetsModules)20 AuthorizationEnforcementModule (co.cask.cdap.security.authorization.AuthorizationEnforcementModule)20 CConfiguration (co.cask.cdap.common.conf.CConfiguration)19 DataFabricModules (co.cask.cdap.data.runtime.DataFabricModules)19 AuthenticationContextModules (co.cask.cdap.security.auth.context.AuthenticationContextModules)16 Injector (com.google.inject.Injector)16 AuthorizationTestModule (co.cask.cdap.security.authorization.AuthorizationTestModule)15 DefaultOwnerAdmin (co.cask.cdap.security.impersonation.DefaultOwnerAdmin)15 ZKClientService (org.apache.twill.zookeeper.ZKClientService)14 TransactionMetricsModule (co.cask.cdap.data.runtime.TransactionMetricsModule)13 UnsupportedUGIProvider (co.cask.cdap.security.impersonation.UnsupportedUGIProvider)13 ExploreClientModule (co.cask.cdap.explore.guice.ExploreClientModule)12 BeforeClass (org.junit.BeforeClass)12 SystemDatasetRuntimeModule (co.cask.cdap.data.runtime.SystemDatasetRuntimeModule)11 ViewAdminModules (co.cask.cdap.data.view.ViewAdminModules)11 Configuration (org.apache.hadoop.conf.Configuration)11