Search in sources :

Example 11 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class LocalStreamFileJanitorTest method init.

@BeforeClass
public static void init() throws IOException {
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new NonCustomLocationUnitTestModule().getModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), Modules.override(new DataSetsModules().getInMemoryModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            // bind to an in mem implementation for this test since the DefaultOwnerStore uses transaction and in this
            // test we are not starting a transaction service
            bind(OwnerStore.class).to(InMemoryOwnerStore.class).in(Scopes.SINGLETON);
        }
    }), new TransactionMetricsModule(), new DataFabricLevelDBModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new StreamAdminModules().getStandaloneModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
        }
    }), new AbstractModule() {

        @Override
        protected void configure() {
            // We don't need notification in this test, hence inject an no-op one
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(NamespaceStore.class).to(InMemoryNamespaceStore.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    });
    locationFactory = injector.getInstance(LocationFactory.class);
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    namespacedLocationFactory = injector.getInstance(NamespacedLocationFactory.class);
    namespaceStore = injector.getInstance(NamespaceStore.class);
    streamAdmin = injector.getInstance(StreamAdmin.class);
    fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
    streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    streamCoordinatorClient.startAndWait();
}
Also used : NamespaceClientRuntimeModule(co.cask.cdap.common.namespace.guice.NamespaceClientRuntimeModule) ConfigModule(co.cask.cdap.common.guice.ConfigModule) UGIProvider(co.cask.cdap.security.impersonation.UGIProvider) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) DataFabricLevelDBModule(co.cask.cdap.data.runtime.DataFabricLevelDBModule) ViewAdminModules(co.cask.cdap.data.view.ViewAdminModules) Injector(com.google.inject.Injector) InMemoryStreamMetaStore(co.cask.cdap.data.stream.service.InMemoryStreamMetaStore) StreamMetaStore(co.cask.cdap.data.stream.service.StreamMetaStore) SystemDatasetRuntimeModule(co.cask.cdap.data.runtime.SystemDatasetRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) NamespaceStore(co.cask.cdap.store.NamespaceStore) InMemoryNamespaceStore(co.cask.cdap.store.InMemoryNamespaceStore) NamespaceAdmin(co.cask.cdap.common.namespace.NamespaceAdmin) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) InMemoryOwnerStore(co.cask.cdap.security.impersonation.InMemoryOwnerStore) AbstractModule(com.google.inject.AbstractModule) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) StreamAdmin(co.cask.cdap.data2.transaction.stream.StreamAdmin) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) NoOpNotificationFeedManager(co.cask.cdap.notifications.feeds.service.NoOpNotificationFeedManager) InMemoryNamespaceStore(co.cask.cdap.store.InMemoryNamespaceStore) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) BeforeClass(org.junit.BeforeClass)

Example 12 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class StreamCoordinatorTestBase method testGeneration.

@Test
public void testGeneration() throws Exception {
    final StreamAdmin streamAdmin = getStreamAdmin();
    final String streamName = "testGen";
    final StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
    streamAdmin.create(streamId);
    StreamCoordinatorClient coordinator = getStreamCoordinator();
    final CountDownLatch genIdChanged = new CountDownLatch(1);
    coordinator.addListener(streamId, new StreamPropertyListener() {

        @Override
        public void generationChanged(StreamId streamId, int generation) {
            if (generation == 10) {
                genIdChanged.countDown();
            }
        }
    });
    // Do concurrent calls to nextGeneration using two threads
    final CyclicBarrier barrier = new CyclicBarrier(2);
    for (int i = 0; i < 2; i++) {
        Thread t = new Thread() {

            @Override
            public void run() {
                try {
                    barrier.await();
                    for (int i = 0; i < 5; i++) {
                        streamAdmin.truncate(streamId);
                    }
                } catch (Exception e) {
                    throw Throwables.propagate(e);
                }
            }
        };
        t.start();
    }
    Assert.assertTrue(genIdChanged.await(10, TimeUnit.SECONDS));
}
Also used : StreamAdmin(co.cask.cdap.data2.transaction.stream.StreamAdmin) StreamId(co.cask.cdap.proto.id.StreamId) CountDownLatch(java.util.concurrent.CountDownLatch) IOException(java.io.IOException) CyclicBarrier(java.util.concurrent.CyclicBarrier) Test(org.junit.Test)

Example 13 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class StreamConsumerTestBase method testNamespacedStreamConsumers.

@Test
public void testNamespacedStreamConsumers() throws Exception {
    // Test two consumers for two streams with the same name, but in different namespaces. Their consumption should be
    // independent of the other.
    String stream = "testNamespacedStreamConsumers";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamId otherStreamId = OTHER_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    streamAdmin.create(streamId);
    streamAdmin.create(otherStreamId);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    StreamConfig otherStreamConfig = streamAdmin.getConfig(otherStreamId);
    // Writes 5 events to both streams
    writeEvents(streamConfig, "Testing ", 5);
    writeEvents(otherStreamConfig, "Testing ", 5);
    streamAdmin.configureInstances(streamId, 0L, 1);
    streamAdmin.configureInstances(otherStreamId, 0L, 1);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    StreamConsumer consumer = consumerFactory.create(streamId, "fifo.rollback", new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
    StreamConsumer otherConsumer = consumerFactory.create(otherStreamId, "fifo.rollback", new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
    // Try to dequeue using both consumers
    TransactionContext context = createTxContext(consumer);
    TransactionContext otherContext = createTxContext(otherConsumer);
    context.start();
    otherContext.start();
    // Consume events from the stream in the default namespace
    DequeueResult<StreamEvent> result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 0", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    context.finish();
    context.start();
    result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 1", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    context.finish();
    context.start();
    result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 2", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    context.finish();
    context.start();
    // Even though a stream with the same name has already consumed 3 events, the otherConsumer is for a stream in a
    // different namespace, so it will still be on the initial event.
    DequeueResult<StreamEvent> result1 = otherConsumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 0", Charsets.UTF_8.decode(result1.iterator().next().getBody()).toString());
    otherContext.finish();
    otherContext.start();
    result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
    result1 = otherConsumer.poll(1, 1, TimeUnit.SECONDS);
    Assert.assertEquals("Testing 3", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
    Assert.assertEquals("Testing 1", Charsets.UTF_8.decode(result1.iterator().next().getBody()).toString());
    // Commit both
    context.finish();
    otherContext.finish();
    consumer.close();
    otherConsumer.close();
}
Also used : StreamId(co.cask.cdap.proto.id.StreamId) TransactionContext(org.apache.tephra.TransactionContext) StreamEvent(co.cask.cdap.api.flow.flowlet.StreamEvent) ConsumerConfig(co.cask.cdap.data2.queue.ConsumerConfig) Test(org.junit.Test)

Example 14 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class HBaseFileStreamAdminTest method init.

@BeforeClass
public static void init() throws Exception {
    InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
    zkServer.startAndWait();
    Configuration hConf = testHBase.getConfiguration();
    addCConfProperties(cConf);
    cConf.setInt(Constants.Stream.CONTAINER_INSTANCES, 1);
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new TransactionMetricsModule(), new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuditModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new DataFabricModules().getDistributedModules(), new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(TransactionStateStorage.class).to(NoOpTransactionStateStorage.class);
            bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    streamAdmin = injector.getInstance(StreamAdmin.class);
    txManager = TxInMemory.getTransactionManager(injector.getInstance(TransactionSystemClient.class));
    fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
    streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    inMemoryAuditPublisher = injector.getInstance(InMemoryAuditPublisher.class);
    authorizer = injector.getInstance(AuthorizerInstantiator.class).get();
    ownerAdmin = injector.getInstance(OwnerAdmin.class);
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
    txManager.startAndWait();
    streamCoordinatorClient.startAndWait();
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) ConfigModule(co.cask.cdap.common.guice.ConfigModule) UGIProvider(co.cask.cdap.security.impersonation.UGIProvider) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) ViewAdminModules(co.cask.cdap.data.view.ViewAdminModules) ZKClientModule(co.cask.cdap.common.guice.ZKClientModule) Injector(com.google.inject.Injector) StreamMetaStore(co.cask.cdap.data.stream.service.StreamMetaStore) InMemoryStreamMetaStore(co.cask.cdap.data.stream.service.InMemoryStreamMetaStore) SimpleNamespaceQueryAdmin(co.cask.cdap.common.namespace.SimpleNamespaceQueryAdmin) NamespaceQueryAdmin(co.cask.cdap.common.namespace.NamespaceQueryAdmin) SystemDatasetRuntimeModule(co.cask.cdap.data.runtime.SystemDatasetRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) NotificationFeedManager(co.cask.cdap.notifications.feeds.NotificationFeedManager) NoOpNotificationFeedManager(co.cask.cdap.notifications.feeds.service.NoOpNotificationFeedManager) InMemoryZKServer(org.apache.twill.internal.zookeeper.InMemoryZKServer) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) OwnerAdmin(co.cask.cdap.security.impersonation.OwnerAdmin) StreamCoordinatorClient(co.cask.cdap.data.stream.StreamCoordinatorClient) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) InMemoryTxSystemClient(org.apache.tephra.inmemory.InMemoryTxSystemClient) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) AbstractModule(com.google.inject.AbstractModule) StreamAdminModules(co.cask.cdap.data.stream.StreamAdminModules) StreamAdmin(co.cask.cdap.data2.transaction.stream.StreamAdmin) StreamFileWriterFactory(co.cask.cdap.data.stream.StreamFileWriterFactory) ZKClientService(org.apache.twill.zookeeper.ZKClientService) InMemoryAuditPublisher(co.cask.cdap.data2.audit.InMemoryAuditPublisher) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) NoOpTransactionStateStorage(org.apache.tephra.persist.NoOpTransactionStateStorage) TransactionStateStorage(org.apache.tephra.persist.TransactionStateStorage) AuditModule(co.cask.cdap.data2.audit.AuditModule) DataFabricModules(co.cask.cdap.data.runtime.DataFabricModules) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) BeforeClass(org.junit.BeforeClass)

Example 15 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class LevelDBFileStreamAdminTest method init.

@BeforeClass
public static void init() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    addCConfProperties(cConf);
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new NonCustomLocationUnitTestModule().getModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new DataSetsModules().getInMemoryModules(), new DataFabricLevelDBModule(), new TransactionMetricsModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuditModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new StreamAdminModules().getStandaloneModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
        }
    }));
    streamAdmin = injector.getInstance(StreamAdmin.class);
    txManager = injector.getInstance(TransactionManager.class);
    fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
    streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    inMemoryAuditPublisher = injector.getInstance(InMemoryAuditPublisher.class);
    authorizer = injector.getInstance(AuthorizerInstantiator.class).get();
    ownerAdmin = injector.getInstance(OwnerAdmin.class);
    streamCoordinatorClient.startAndWait();
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
    txManager.startAndWait();
}
Also used : ConfigModule(co.cask.cdap.common.guice.ConfigModule) UGIProvider(co.cask.cdap.security.impersonation.UGIProvider) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) DataFabricLevelDBModule(co.cask.cdap.data.runtime.DataFabricLevelDBModule) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) ViewAdminModules(co.cask.cdap.data.view.ViewAdminModules) Injector(com.google.inject.Injector) InMemoryStreamMetaStore(co.cask.cdap.data.stream.service.InMemoryStreamMetaStore) StreamMetaStore(co.cask.cdap.data.stream.service.StreamMetaStore) NamespaceQueryAdmin(co.cask.cdap.common.namespace.NamespaceQueryAdmin) SimpleNamespaceQueryAdmin(co.cask.cdap.common.namespace.SimpleNamespaceQueryAdmin) SystemDatasetRuntimeModule(co.cask.cdap.data.runtime.SystemDatasetRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) NotificationFeedManager(co.cask.cdap.notifications.feeds.NotificationFeedManager) NoOpNotificationFeedManager(co.cask.cdap.notifications.feeds.service.NoOpNotificationFeedManager) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) OwnerAdmin(co.cask.cdap.security.impersonation.OwnerAdmin) StreamCoordinatorClient(co.cask.cdap.data.stream.StreamCoordinatorClient) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) CConfiguration(co.cask.cdap.common.conf.CConfiguration) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) AbstractModule(com.google.inject.AbstractModule) StreamAdminModules(co.cask.cdap.data.stream.StreamAdminModules) StreamAdmin(co.cask.cdap.data2.transaction.stream.StreamAdmin) StreamFileWriterFactory(co.cask.cdap.data.stream.StreamFileWriterFactory) InMemoryAuditPublisher(co.cask.cdap.data2.audit.InMemoryAuditPublisher) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) TransactionManager(org.apache.tephra.TransactionManager) AuditModule(co.cask.cdap.data2.audit.AuditModule) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) BeforeClass(org.junit.BeforeClass)

Aggregations

StreamAdmin (co.cask.cdap.data2.transaction.stream.StreamAdmin)25 StreamId (co.cask.cdap.proto.id.StreamId)18 Test (org.junit.Test)16 NamespacedLocationFactory (co.cask.cdap.common.namespace.NamespacedLocationFactory)14 Injector (com.google.inject.Injector)14 ConfigModule (co.cask.cdap.common.guice.ConfigModule)13 ExploreClientModule (co.cask.cdap.explore.guice.ExploreClientModule)13 AuthorizationEnforcementModule (co.cask.cdap.security.authorization.AuthorizationEnforcementModule)13 StreamEvent (co.cask.cdap.api.flow.flowlet.StreamEvent)12 DiscoveryRuntimeModule (co.cask.cdap.common.guice.DiscoveryRuntimeModule)12 DataSetsModules (co.cask.cdap.data.runtime.DataSetsModules)12 ViewAdminModules (co.cask.cdap.data.view.ViewAdminModules)12 AuthenticationContextModules (co.cask.cdap.security.auth.context.AuthenticationContextModules)12 AuthorizationTestModule (co.cask.cdap.security.authorization.AuthorizationTestModule)12 AbstractModule (com.google.inject.AbstractModule)12 BeforeClass (org.junit.BeforeClass)12 CConfiguration (co.cask.cdap.common.conf.CConfiguration)11 StreamMetaStore (co.cask.cdap.data.stream.service.StreamMetaStore)11 DefaultOwnerAdmin (co.cask.cdap.security.impersonation.DefaultOwnerAdmin)11 Configuration (org.apache.hadoop.conf.Configuration)11