use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class ConcurrentStreamWriterTestBase method testConcurrentWrite.
@Test
public void testConcurrentWrite() throws Exception {
final String streamName = "testConcurrentWrite";
NamespaceId namespace = new NamespaceId("namespace");
StreamId streamId = namespace.stream(streamName);
StreamAdmin streamAdmin = new TestStreamAdmin(getNamespacedLocationFactory(), Long.MAX_VALUE, 1000);
int threads = Runtime.getRuntime().availableProcessors() * 4;
StreamFileWriterFactory fileWriterFactory = createStreamFileWriterFactory();
final ConcurrentStreamWriter streamWriter = createStreamWriter(streamId, streamAdmin, threads, fileWriterFactory);
// Starts n threads to write events through stream writer, each thread write 1000 events
final int msgPerThread = 1000;
final CountDownLatch startLatch = new CountDownLatch(1);
final CountDownLatch completion = new CountDownLatch(threads);
ExecutorService executor = Executors.newFixedThreadPool(threads);
// Half of the threads write events one by one, the other half writes in batch of size 10
for (int i = 0; i < threads / 2; i++) {
executor.execute(createWriterTask(streamId, streamWriter, i, msgPerThread, 1, startLatch, completion));
}
for (int i = threads / 2; i < threads; i++) {
executor.execute(createWriterTask(streamId, streamWriter, i, msgPerThread, 10, startLatch, completion));
}
startLatch.countDown();
Assert.assertTrue(completion.await(120, TimeUnit.SECONDS));
// Verify all events are written.
// There should be only one partition and one file inside
Location partitionLocation = streamAdmin.getConfig(streamId).getLocation().list().get(0);
Location streamLocation = StreamUtils.createStreamLocation(partitionLocation, fileWriterFactory.getFileNamePrefix(), 0, StreamFileType.EVENT);
StreamDataFileReader reader = StreamDataFileReader.create(Locations.newInputSupplier(streamLocation));
List<StreamEvent> events = Lists.newArrayListWithCapacity(threads * msgPerThread);
// Should read all messages
Assert.assertEquals(threads * msgPerThread, reader.read(events, Integer.MAX_VALUE, 0, TimeUnit.SECONDS));
// Verify all messages as expected
Assert.assertTrue(verifyEvents(threads, msgPerThread, events));
reader.close();
streamWriter.close();
}
use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class StreamConsumerTestBase method testTTLMultipleEventsWithSameTimestamp.
@Test
public void testTTLMultipleEventsWithSameTimestamp() throws Exception {
String stream = "testTTLMultipleEventsWithSameTimestamp";
StreamId streamId = TEST_NAMESPACE.stream(stream);
StreamAdmin streamAdmin = getStreamAdmin();
// Create stream with ttl of 1 day
final long ttl = TimeUnit.DAYS.toMillis(1);
final long currentTime = System.currentTimeMillis();
final long increment = TimeUnit.SECONDS.toMillis(1);
final long approxEarliestNonExpiredTime = currentTime - TimeUnit.HOURS.toMillis(1);
Properties streamProperties = new Properties();
streamProperties.setProperty(Constants.Stream.TTL, Long.toString(ttl));
streamProperties.setProperty(Constants.Stream.PARTITION_DURATION, Long.toString(ttl));
streamAdmin.create(streamId, streamProperties);
StreamConfig streamConfig = streamAdmin.getConfig(streamId);
streamAdmin.configureInstances(streamId, 0L, 1);
StreamConsumerFactory consumerFactory = getConsumerFactory();
Assert.assertEquals(ttl, streamConfig.getTTL());
Assert.assertEquals(ttl, streamConfig.getPartitionDuration());
// Write 100 expired messages to stream with expired timestamp
writeEvents(streamConfig, "Old event ", 10, new ConstantClock(0));
// Write 500 non-expired messages to stream with timestamp approxEarliestNonExpiredTime..currentTime
Set<StreamEvent> expectedEvents = Sets.newTreeSet(STREAM_EVENT_COMPARATOR);
FileWriter<StreamEvent> writer = getFileWriterFactory().create(streamConfig, 0);
try {
expectedEvents.addAll(writeEvents(writer, "New event pre-flush ", 20, new IncrementingClock(approxEarliestNonExpiredTime, increment, 5)));
writer.flush();
expectedEvents.addAll(writeEvents(writer, "New event post-flush ", 20, new IncrementingClock(approxEarliestNonExpiredTime + 1, increment, 5)));
} finally {
writer.close();
}
StreamConsumer consumer = consumerFactory.create(streamId, stream, new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
verifyEvents(consumer, expectedEvents);
TransactionContext txContext = createTxContext(consumer);
txContext.start();
try {
// Should be no more pending events
DequeueResult<StreamEvent> result = consumer.poll(1, 1, TimeUnit.SECONDS);
Assert.assertTrue(result.isEmpty());
} finally {
txContext.finish();
}
consumer.close();
}
use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class StreamConsumerTestBase method testFIFOReconfigure.
@Test
public void testFIFOReconfigure() throws Exception {
String stream = "testReconfigure";
StreamId streamId = TEST_NAMESPACE.stream(stream);
StreamAdmin streamAdmin = getStreamAdmin();
streamAdmin.create(streamId);
StreamConfig streamConfig = streamAdmin.getConfig(streamId);
// Writes 5 events
writeEvents(streamConfig, "Testing ", 5);
// Configure 3 consumers.
streamAdmin.configureInstances(streamId, 0L, 3);
StreamConsumerFactory consumerFactory = getConsumerFactory();
// Starts three consumers
List<StreamConsumer> consumers = Lists.newArrayList();
for (int i = 0; i < 3; i++) {
consumers.add(consumerFactory.create(streamId, "fifo.reconfigure", new ConsumerConfig(0L, i, 3, DequeueStrategy.FIFO, null)));
}
List<TransactionContext> txContexts = Lists.newArrayList();
for (StreamConsumer consumer : consumers) {
txContexts.add(createTxContext(consumer));
}
for (TransactionContext txContext : txContexts) {
txContext.start();
}
// Consumer an item from each consumer, but only have the first one commit.
for (int i = 0; i < consumers.size(); i++) {
DequeueResult<StreamEvent> result = consumers.get(i).poll(1, 1, TimeUnit.SECONDS);
Assert.assertEquals("Testing " + i, Charsets.UTF_8.decode(result.iterator().next().getBody()).toString());
if (i == 0) {
txContexts.get(i).finish();
} else {
txContexts.get(i).abort();
}
}
for (StreamConsumer consumer : consumers) {
consumer.close();
}
// Reconfigure to have two consumers.
streamAdmin.configureInstances(streamId, 0L, 2);
consumers.clear();
for (int i = 0; i < 2; i++) {
consumers.add(consumerFactory.create(streamId, "fifo.reconfigure", new ConsumerConfig(0L, i, 2, DequeueStrategy.FIFO, null)));
}
txContexts.clear();
for (StreamConsumer consumer : consumers) {
txContexts.add(createTxContext(consumer));
}
// Consumer an item from each consumer, they should see all four items.
Set<String> messages = Sets.newTreeSet();
boolean done;
do {
for (TransactionContext txContext : txContexts) {
txContext.start();
}
done = true;
for (int i = 0; i < consumers.size(); i++) {
DequeueResult<StreamEvent> result = consumers.get(i).poll(1, 1, TimeUnit.SECONDS);
if (result.isEmpty()) {
continue;
}
done = false;
messages.add(Charsets.UTF_8.decode(result.iterator().next().getBody()).toString());
txContexts.get(i).finish();
}
} while (!done);
Assert.assertEquals(4, messages.size());
int count = 1;
for (String msg : messages) {
Assert.assertEquals("Testing " + count, msg);
count++;
}
for (StreamConsumer consumer : consumers) {
consumer.close();
}
}
use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class NamespaceHttpHandlerTest method testDeleteAll.
@Test
public void testDeleteAll() throws Exception {
CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
// test deleting non-existent namespace
assertResponseCode(404, deleteNamespace("doesnotexist"));
assertResponseCode(200, createNamespace(NAME));
assertResponseCode(200, getNamespace(NAME));
assertResponseCode(200, createNamespace(OTHER_NAME));
assertResponseCode(200, getNamespace(OTHER_NAME));
NamespacedLocationFactory namespacedLocationFactory = getInjector().getInstance(NamespacedLocationFactory.class);
Location nsLocation = namespacedLocationFactory.get(new NamespaceId(NAME));
Assert.assertTrue(nsLocation.exists());
DatasetFramework dsFramework = getInjector().getInstance(DatasetFramework.class);
StreamAdmin streamAdmin = getInjector().getInstance(StreamAdmin.class);
deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
deploy(AppWithDataset.class, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
deploy(AppWithStreamSizeSchedule.class, Constants.Gateway.API_VERSION_3_TOKEN, OTHER_NAME);
deploy(AppForUnrecoverableResetTest.class, Constants.Gateway.API_VERSION_3_TOKEN, OTHER_NAME);
DatasetId myDataset = new DatasetId(NAME, "myds");
StreamId myStream = new StreamId(OTHER_NAME, "stream");
Assert.assertTrue(dsFramework.hasInstance(myDataset));
Assert.assertTrue(streamAdmin.exists(myStream));
Id.Program program = Id.Program.from(NAME_ID, "AppWithServices", ProgramType.SERVICE, "NoOpService");
startProgram(program);
boolean resetEnabled = cConf.getBoolean(Constants.Dangerous.UNRECOVERABLE_RESET);
cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, false);
// because unrecoverable reset is disabled
assertResponseCode(403, deleteNamespace(NAME));
cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, resetEnabled);
// because service is running
assertResponseCode(409, deleteNamespace(NAME));
Assert.assertTrue(nsLocation.exists());
stopProgram(program);
// delete should work now
assertResponseCode(200, deleteNamespace(NAME));
Assert.assertFalse(nsLocation.exists());
Assert.assertFalse(dsFramework.hasInstance(myDataset));
Assert.assertTrue(streamAdmin.exists(myStream));
assertResponseCode(200, deleteNamespace(OTHER_NAME));
Assert.assertFalse(streamAdmin.exists(myStream));
// Create the namespace again and deploy the application containing schedules.
// Application deployment should succeed.
assertResponseCode(200, createNamespace(OTHER_NAME));
HttpResponse response = deploy(AppForUnrecoverableResetTest.class, Constants.Gateway.API_VERSION_3_TOKEN, OTHER_NAME);
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
assertResponseCode(200, deleteNamespace(OTHER_NAME));
}
use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.
the class LevelDBStreamConsumerTest method init.
@BeforeClass
public static void init() throws Exception {
cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
Injector injector = Guice.createInjector(new ConfigModule(cConf), new NonCustomLocationUnitTestModule().getModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new DataSetsModules().getInMemoryModules(), new DataFabricLevelDBModule(), new TransactionMetricsModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new StreamAdminModules().getStandaloneModules()).with(new AbstractModule() {
@Override
protected void configure() {
bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
}
}));
consumerFactory = injector.getInstance(StreamConsumerFactory.class);
streamAdmin = injector.getInstance(StreamAdmin.class);
txClient = injector.getInstance(TransactionSystemClient.class);
txManager = injector.getInstance(TransactionManager.class);
queueClientFactory = injector.getInstance(QueueClientFactory.class);
fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
streamCoordinatorClient.startAndWait();
txManager.startAndWait();
setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
}
Aggregations