use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class LocalStreamFileJanitorTest method init.
@BeforeClass
public static void init() throws IOException {
cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
Injector injector = Guice.createInjector(new ConfigModule(cConf), new NonCustomLocationUnitTestModule().getModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), Modules.override(new DataSetsModules().getInMemoryModules()).with(new AbstractModule() {
@Override
protected void configure() {
// bind to an in mem implementation for this test since the DefaultOwnerStore uses transaction and in this
// test we are not starting a transaction service
bind(OwnerStore.class).to(InMemoryOwnerStore.class).in(Scopes.SINGLETON);
}
}), new TransactionMetricsModule(), new DataFabricLevelDBModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new StreamAdminModules().getStandaloneModules()).with(new AbstractModule() {
@Override
protected void configure() {
bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
}
}), new AbstractModule() {
@Override
protected void configure() {
// We don't need notification in this test, hence inject an no-op one
bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
bind(NamespaceStore.class).to(InMemoryNamespaceStore.class);
bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
}
});
locationFactory = injector.getInstance(LocationFactory.class);
namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
namespacedLocationFactory = injector.getInstance(NamespacedLocationFactory.class);
namespaceStore = injector.getInstance(NamespaceStore.class);
streamAdmin = injector.getInstance(StreamAdmin.class);
fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
streamCoordinatorClient.startAndWait();
}
use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class StreamCoordinatorTestBase method testGeneration.
@Test
public void testGeneration() throws Exception {
final StreamAdmin streamAdmin = getStreamAdmin();
final String streamName = "testGen";
final StreamId streamId = NamespaceId.DEFAULT.stream(streamName);
streamAdmin.create(streamId);
StreamCoordinatorClient coordinator = getStreamCoordinator();
final CountDownLatch genIdChanged = new CountDownLatch(1);
coordinator.addListener(streamId, new StreamPropertyListener() {
@Override
public void generationChanged(StreamId streamId, int generation) {
if (generation == 10) {
genIdChanged.countDown();
}
}
});
// Do concurrent calls to nextGeneration using two threads
final CyclicBarrier barrier = new CyclicBarrier(2);
for (int i = 0; i < 2; i++) {
Thread t = new Thread() {
@Override
public void run() {
try {
barrier.await();
for (int i = 0; i < 5; i++) {
streamAdmin.truncate(streamId);
}
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
};
t.start();
}
Assert.assertTrue(genIdChanged.await(10, TimeUnit.SECONDS));
}
use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class StreamConsumerTestBase method testNamespacedStreamConsumers.
@Test
public void testNamespacedStreamConsumers() throws Exception {
// Test two consumers for two streams with the same name, but in different namespaces. Their consumption should be
// independent of the other.
String stream = "testNamespacedStreamConsumers";
StreamId streamId = TEST_NAMESPACE.stream(stream);
StreamId otherStreamId = OTHER_NAMESPACE.stream(stream);
StreamAdmin streamAdmin = getStreamAdmin();
streamAdmin.create(streamId);
streamAdmin.create(otherStreamId);
StreamConfig streamConfig = streamAdmin.getConfig(streamId);
StreamConfig otherStreamConfig = streamAdmin.getConfig(otherStreamId);
// Writes 5 events to both streams
writeEvents(streamConfig, "Testing ", 5);
writeEvents(otherStreamConfig, "Testing ", 5);
streamAdmin.configureInstances(streamId, 0L, 1);
streamAdmin.configureInstances(otherStreamId, 0L, 1);
StreamConsumerFactory consumerFactory = getConsumerFactory();
StreamConsumer consumer = consumerFactory.create(streamId, "fifo.rollback", new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
StreamConsumer otherConsumer = consumerFactory.create(otherStreamId, "fifo.rollback", new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
// Try to dequeue using both consumers
TransactionContext context = createTxContext(consumer);
TransactionContext otherContext = createTxContext(otherConsumer);
context.start();
otherContext.start();
// Consume events from the stream in the default namespace
DequeueResult<StreamEvent> result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
Assert.assertEquals("Testing 0", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
context.finish();
context.start();
result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
Assert.assertEquals("Testing 1", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
context.finish();
context.start();
result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
Assert.assertEquals("Testing 2", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
context.finish();
context.start();
// Even though a stream with the same name has already consumed 3 events, the otherConsumer is for a stream in a
// different namespace, so it will still be on the initial event.
DequeueResult<StreamEvent> result1 = otherConsumer.poll(1, 1, TimeUnit.SECONDS);
Assert.assertEquals("Testing 0", Charsets.UTF_8.decode(result1.iterator().next().getBody()).toString());
otherContext.finish();
otherContext.start();
result0 = consumer.poll(1, 1, TimeUnit.SECONDS);
result1 = otherConsumer.poll(1, 1, TimeUnit.SECONDS);
Assert.assertEquals("Testing 3", Charsets.UTF_8.decode(result0.iterator().next().getBody()).toString());
Assert.assertEquals("Testing 1", Charsets.UTF_8.decode(result1.iterator().next().getBody()).toString());
// Commit both
context.finish();
otherContext.finish();
consumer.close();
otherConsumer.close();
}
use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class HBaseFileStreamAdminTest method init.
@BeforeClass
public static void init() throws Exception {
InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
zkServer.startAndWait();
Configuration hConf = testHBase.getConfiguration();
addCConfProperties(cConf);
cConf.setInt(Constants.Stream.CONTAINER_INSTANCES, 1);
cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
Injector injector = Guice.createInjector(new ConfigModule(cConf, hConf), new ZKClientModule(), new NonCustomLocationUnitTestModule().getModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new TransactionMetricsModule(), new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuditModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new DataFabricModules().getDistributedModules(), new StreamAdminModules().getDistributedModules()).with(new AbstractModule() {
@Override
protected void configure() {
bind(TransactionStateStorage.class).to(NoOpTransactionStateStorage.class);
bind(TransactionSystemClient.class).to(InMemoryTxSystemClient.class).in(Singleton.class);
bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
}
}));
ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
zkClientService.startAndWait();
streamAdmin = injector.getInstance(StreamAdmin.class);
txManager = TxInMemory.getTransactionManager(injector.getInstance(TransactionSystemClient.class));
fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
inMemoryAuditPublisher = injector.getInstance(InMemoryAuditPublisher.class);
authorizer = injector.getInstance(AuthorizerInstantiator.class).get();
ownerAdmin = injector.getInstance(OwnerAdmin.class);
setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
txManager.startAndWait();
streamCoordinatorClient.startAndWait();
}
use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class LevelDBFileStreamAdminTest method init.
@BeforeClass
public static void init() throws Exception {
CConfiguration cConf = CConfiguration.create();
cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
addCConfProperties(cConf);
Injector injector = Guice.createInjector(new ConfigModule(cConf), new NonCustomLocationUnitTestModule().getModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new DataSetsModules().getInMemoryModules(), new DataFabricLevelDBModule(), new TransactionMetricsModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new AuditModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new StreamAdminModules().getStandaloneModules()).with(new AbstractModule() {
@Override
protected void configure() {
bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
}
}));
streamAdmin = injector.getInstance(StreamAdmin.class);
txManager = injector.getInstance(TransactionManager.class);
fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
inMemoryAuditPublisher = injector.getInstance(InMemoryAuditPublisher.class);
authorizer = injector.getInstance(AuthorizerInstantiator.class).get();
ownerAdmin = injector.getInstance(OwnerAdmin.class);
streamCoordinatorClient.startAndWait();
setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
txManager.startAndWait();
}
Aggregations