Examples with TransactionMetricsModule io.cdap.cdap.data.runtime.TransactionMetricsModule used on opensource projects

Example 6 with TransactionMetricsModule

use of io.cdap.cdap.data.runtime.TransactionMetricsModule in project cdap by caskdata.

the class InMemoryMetricsTableTest method setup.

@BeforeClass
public static void setup() {
    Injector injector = Guice.createInjector(new ConfigModule(), new NonCustomLocationUnitTestModule(), new InMemoryDiscoveryModule(), new DataFabricModules().getInMemoryModules(), new TransactionMetricsModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new PrivateModule() {

        @Override
        protected void configure() {
            bind(DatasetDefinitionRegistryFactory.class).to(DefaultDatasetDefinitionRegistryFactory.class).in(Scopes.SINGLETON);
            bind(DatasetFramework.class).to(InMemoryDatasetFramework.class);
            expose(DatasetFramework.class);
        }
    });
    dsFramework = injector.getInstance(DatasetFramework.class);
}

Example 7 with TransactionMetricsModule

use of io.cdap.cdap.data.runtime.TransactionMetricsModule in project cdap by cdapio.

the class TransactionServiceClientTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    HBaseTestingUtility hBaseTestingUtility = new HBaseTestingUtility();
    hBaseTestingUtility.startMiniDFSCluster(1);
    Configuration hConf = hBaseTestingUtility.getConfiguration();
    hConf.setBoolean("fs.hdfs.impl.disable.cache", true);
    zkServer = InMemoryZKServer.builder().build();
    zkServer.startAndWait();
    CConfiguration cConf = CConfiguration.create();
    // tests should use the current user for HDFS
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    cConf.setBoolean(TxConstants.TransactionPruning.PRUNE_ENABLE, false);
    // we want persisting for this test
    cConf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, true);
    // getCommonConfiguration() sets up an hConf with tx service configuration.
    // however, createTxService() will override these with defaults from the CConf.
    // hence, we must pass in these settings when creating the tx service.
    Configuration extraCConf = new Configuration();
    extraCConf.clear();
    extraCConf = getCommonConfiguration(extraCConf);
    for (Map.Entry<String, String> entry : extraCConf) {
        cConf.set(entry.getKey(), entry.getValue());
    }
    server = TransactionServiceTest.createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder(), cConf);
    server.startAndWait();
    injector = Guice.createInjector(new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getNoOpModule(), new ZKClientModule(), new ZKDiscoveryModule(), new NonCustomLocationUnitTestModule(), new TransactionMetricsModule(), new DataFabricModules().getDistributedModules(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }, Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetadataStorage.class).to(NoopMetadataStorage.class);
        }
    }), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule());
    zkClient = injector.getInstance(ZKClientService.class);
    zkClient.startAndWait();
    txStateStorage = injector.getInstance(TransactionStateStorage.class);
    txStateStorage.startAndWait();
}

Example 8 with TransactionMetricsModule

use of io.cdap.cdap.data.runtime.TransactionMetricsModule in project cdap by cdapio.

the class TransactionServiceTest method testHA.

@Test(timeout = 30000)
public void testHA() throws Exception {
    // NOTE: we play with blocking/nonblocking a lot below
    // as until we integrate with "leader election" stuff, service blocks on start if it is not a leader
    // TODO: fix this by integration with generic leader election stuff
    CConfiguration cConf = CConfiguration.create();
    // tests should use the current user for HDFS
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), RemoteAuthenticatorModules.getNoOpModule(), new ZKClientModule(), new ZKDiscoveryModule(), new NonCustomLocationUnitTestModule(), new TransactionMetricsModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }, new DataFabricModules().getDistributedModules(), Modules.override(new DataSetsModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetadataStorage.class).to(NoopMetadataStorage.class);
        }
    }), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule());
    ZKClientService zkClient = injector.getInstance(ZKClientService.class);
    zkClient.startAndWait();
    try {
        final Table table = createTable("myTable");
        // tx service client
        // NOTE: we can init it earlier than we start services, it should pick them up when they are available
        TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
        TransactionExecutor txExecutor = new DefaultTransactionExecutor(txClient, ImmutableList.of((TransactionAware) table));
        // starting tx service, tx client can pick it up
        TransactionService first = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder());
        first.startAndWait();
        Assert.assertNotNull(txClient.startShort());
        verifyGetAndPut(table, txExecutor, null, "val1");
        // starting another tx service should not hurt
        TransactionService second = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder());
        // NOTE: we don't have to wait for start as client should pick it up anyways, but we do wait to ensure
        // the case with two active is handled well
        second.startAndWait();
        // wait for affect a bit
        TimeUnit.SECONDS.sleep(1);
        Assert.assertNotNull(txClient.startShort());
        verifyGetAndPut(table, txExecutor, "val1", "val2");
        // shutting down the first one is fine: we have another one to pick up the leader role
        first.stopAndWait();
        Assert.assertNotNull(txClient.startShort());
        verifyGetAndPut(table, txExecutor, "val2", "val3");
        // doing same trick again to failover to the third one
        TransactionService third = createTxService(zkServer.getConnectionStr(), Networks.getRandomPort(), hConf, tmpFolder.newFolder());
        // NOTE: we don't have to wait for start as client should pick it up anyways
        third.start();
        // stopping second one
        second.stopAndWait();
        Assert.assertNotNull(txClient.startShort());
        verifyGetAndPut(table, txExecutor, "val3", "val4");
        // releasing resources
        third.stop();
    } finally {
        try {
            dropTable("myTable");
        } finally {
            zkClient.stopAndWait();
        }
    }
}

Example 9 with TransactionMetricsModule

use of io.cdap.cdap.data.runtime.TransactionMetricsModule in project cdap by cdapio.

the class LocalLogAppenderResilientTest method testResilientLogging.

@Test
public void testResilientLogging() throws Exception {
    Configuration hConf = new Configuration();
    CConfiguration cConf = CConfiguration.create();
    File datasetDir = new File(tmpFolder.newFolder(), "datasetUser");
    // noinspection ResultOfMethodCallIgnored
    datasetDir.mkdirs();
    cConf.set(Constants.Dataset.Manager.OUTPUT_DIR, datasetDir.getAbsolutePath());
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, "localhost");
    cConf.set(Constants.Dataset.Executor.ADDRESS, "localhost");
    cConf.setInt(Constants.Dataset.Executor.PORT, Networks.getRandomPort());
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getNoOpModule(), new IOModule(), new ZKClientModule(), new KafkaClientModule(), new InMemoryDiscoveryModule(), new NonCustomLocationUnitTestModule(), new DataFabricModules().getInMemoryModules(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new TransactionMetricsModule(), new ExploreClientModule(), new LocalLogAppenderModule(), new NamespaceAdminTestModule(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getMasterModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(NoOpOwnerAdmin.class);
            bind(MetadataServiceClient.class).to(NoOpMetadataServiceClient.class);
        }
    });
    TransactionManager txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    DatasetOpExecutorService opExecutorService = injector.getInstance(DatasetOpExecutorService.class);
    opExecutorService.startAndWait();
    // Start the logging before starting the service.
    LoggingContextAccessor.setLoggingContext(new WorkerLoggingContext("TRL_ACCT_1", "APP_1", "WORKER_1", "RUN", "INSTANCE"));
    String logBaseDir = "trl-log/log_files_" + new Random(System.currentTimeMillis()).nextLong();
    cConf.set(LoggingConfiguration.LOG_BASE_DIR, logBaseDir);
    cConf.setInt(LoggingConfiguration.LOG_MAX_FILE_SIZE_BYTES, 20 * 1024);
    final LogAppender appender = injector.getInstance(LocalLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestResilientLogging");
    int failureMsgCount = 3;
    final CountDownLatch failureLatch = new CountDownLatch(failureMsgCount);
    LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
    loggerContext.getStatusManager().add(new StatusListener() {

        @Override
        public void addStatusEvent(Status status) {
            if (status.getLevel() != Status.ERROR || status.getOrigin() != appender) {
                return;
            }
            Throwable cause = status.getThrowable();
            if (cause != null) {
                Throwable rootCause = Throwables.getRootCause(cause);
                if (rootCause instanceof ServiceUnavailableException) {
                    String serviceName = ((ServiceUnavailableException) rootCause).getServiceName();
                    if (Constants.Service.DATASET_MANAGER.equals(serviceName)) {
                        failureLatch.countDown();
                    }
                }
            }
        }
    });
    Logger logger = LoggerFactory.getLogger("TestResilientLogging");
    for (int i = 0; i < failureMsgCount; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    // Wait for the three failure to append to happen
    // The wait time has to be > 3 seconds because DatasetServiceClient has 1 second timeout on discovery
    failureLatch.await(5, TimeUnit.SECONDS);
    // Start dataset service, wait for it to be discoverable
    DatasetService dsService = injector.getInstance(DatasetService.class);
    dsService.startAndWait();
    final CountDownLatch startLatch = new CountDownLatch(1);
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    discoveryClient.discover(Constants.Service.DATASET_MANAGER).watchChanges(new ServiceDiscovered.ChangeListener() {

        @Override
        public void onChange(ServiceDiscovered serviceDiscovered) {
            if (!Iterables.isEmpty(serviceDiscovered)) {
                startLatch.countDown();
            }
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    startLatch.await(5, TimeUnit.SECONDS);
    // Do some more logging after the service is started.
    for (int i = 5; i < 10; ++i) {
        Exception e1 = new Exception("Test Exception1");
        Exception e2 = new Exception("Test Exception2", e1);
        logger.warn("Test log message " + i + " {} {}", "arg1", "arg2", e2);
    }
    appender.stop();
    // Verify - we should have at least 5 events.
    LoggingContext loggingContext = new WorkerLoggingContext("TRL_ACCT_1", "APP_1", "WORKER_1", "RUN", "INSTANCE");
    FileLogReader logTail = injector.getInstance(FileLogReader.class);
    LoggingTester.LogCallback logCallback1 = new LoggingTester.LogCallback();
    logTail.getLogPrev(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> allEvents = logCallback1.getEvents();
    Assert.assertTrue(allEvents.toString(), allEvents.size() >= 5);
    // Finally - stop all services
    Services.chainStop(dsService, opExecutorService, txManager);
}

Example 10 with TransactionMetricsModule

use of io.cdap.cdap.data.runtime.TransactionMetricsModule in project cdap by cdapio.

the class HBaseMetricsTableTest method setup.

@BeforeClass
public static void setup() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_HDFS_USER, System.getProperty("user.name"));
    Injector injector = Guice.createInjector(new DataFabricModules().getDistributedModules(), new ConfigModule(cConf, TEST_HBASE.getConfiguration()), new ZKClientModule(), new ZKDiscoveryModule(), new TransactionMetricsModule(), new DFSLocationModule(), new NamespaceAdminTestModule(), new SystemDatasetRuntimeModule().getDistributedModules(), new DataSetsModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    });
    dsFramework = injector.getInstance(DatasetFramework.class);
    tableUtil = injector.getInstance(HBaseTableUtil.class);
    ddlExecutor = new HBaseDDLExecutorFactory(cConf, TEST_HBASE.getHBaseAdmin().getConfiguration()).get();
    ddlExecutor.createNamespaceIfNotExists(tableUtil.getHBaseNamespace(NamespaceId.SYSTEM));
}