Search in sources :

Example 26 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class ConcurrentStreamWriterTestBase method testConcurrentWrite.

@Test
public void testConcurrentWrite() throws Exception {
    final String streamName = "testConcurrentWrite";
    NamespaceId namespace = new NamespaceId("namespace");
    StreamId streamId = namespace.stream(streamName);
    StreamAdmin streamAdmin = new TestStreamAdmin(getNamespacedLocationFactory(), Long.MAX_VALUE, 1000);
    int threads = Runtime.getRuntime().availableProcessors() * 4;
    StreamFileWriterFactory fileWriterFactory = createStreamFileWriterFactory();
    final ConcurrentStreamWriter streamWriter = createStreamWriter(streamId, streamAdmin, threads, fileWriterFactory);
    // Starts n threads to write events through stream writer, each thread write 1000 events
    final int msgPerThread = 1000;
    final CountDownLatch startLatch = new CountDownLatch(1);
    final CountDownLatch completion = new CountDownLatch(threads);
    ExecutorService executor = Executors.newFixedThreadPool(threads);
    // Half of the threads write events one by one, the other half writes in batch of size 10
    for (int i = 0; i < threads / 2; i++) {
        executor.execute(createWriterTask(streamId, streamWriter, i, msgPerThread, 1, startLatch, completion));
    }
    for (int i = threads / 2; i < threads; i++) {
        executor.execute(createWriterTask(streamId, streamWriter, i, msgPerThread, 10, startLatch, completion));
    }
    startLatch.countDown();
    Assert.assertTrue(completion.await(120, TimeUnit.SECONDS));
    // Verify all events are written.
    // There should be only one partition and one file inside
    Location partitionLocation = streamAdmin.getConfig(streamId).getLocation().list().get(0);
    Location streamLocation = StreamUtils.createStreamLocation(partitionLocation, fileWriterFactory.getFileNamePrefix(), 0, StreamFileType.EVENT);
    StreamDataFileReader reader = StreamDataFileReader.create(Locations.newInputSupplier(streamLocation));
    List<StreamEvent> events = Lists.newArrayListWithCapacity(threads * msgPerThread);
    // Should read all messages
    Assert.assertEquals(threads * msgPerThread, reader.read(events, Integer.MAX_VALUE, 0, TimeUnit.SECONDS));
    // Verify all messages as expected
    Assert.assertTrue(verifyEvents(threads, msgPerThread, events));
    reader.close();
    streamWriter.close();
}
Also used : StreamId(co.cask.cdap.proto.id.StreamId) StreamEvent(co.cask.cdap.api.flow.flowlet.StreamEvent) CountDownLatch(java.util.concurrent.CountDownLatch) StreamAdmin(co.cask.cdap.data2.transaction.stream.StreamAdmin) NoopStreamAdmin(co.cask.cdap.data.stream.NoopStreamAdmin) StreamFileWriterFactory(co.cask.cdap.data.stream.StreamFileWriterFactory) LocationStreamFileWriterFactory(co.cask.cdap.data.runtime.LocationStreamFileWriterFactory) ExecutorService(java.util.concurrent.ExecutorService) NamespaceId(co.cask.cdap.proto.id.NamespaceId) StreamDataFileReader(co.cask.cdap.data.stream.StreamDataFileReader) Location(org.apache.twill.filesystem.Location) Test(org.junit.Test)

Example 27 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class StreamConsumerTestBase method testTTLMultipleEventsWithSameTimestamp.

@Test
public void testTTLMultipleEventsWithSameTimestamp() throws Exception {
    String stream = "testTTLMultipleEventsWithSameTimestamp";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    // Create stream with ttl of 1 day
    final long ttl = TimeUnit.DAYS.toMillis(1);
    final long currentTime = System.currentTimeMillis();
    final long increment = TimeUnit.SECONDS.toMillis(1);
    final long approxEarliestNonExpiredTime = currentTime - TimeUnit.HOURS.toMillis(1);
    Properties streamProperties = new Properties();
    streamProperties.setProperty(Constants.Stream.TTL, Long.toString(ttl));
    streamProperties.setProperty(Constants.Stream.PARTITION_DURATION, Long.toString(ttl));
    streamAdmin.create(streamId, streamProperties);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    streamAdmin.configureInstances(streamId, 0L, 1);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    Assert.assertEquals(ttl, streamConfig.getTTL());
    Assert.assertEquals(ttl, streamConfig.getPartitionDuration());
    // Write 100 expired messages to stream with expired timestamp
    writeEvents(streamConfig, "Old event ", 10, new ConstantClock(0));
    // Write 500 non-expired messages to stream with timestamp approxEarliestNonExpiredTime..currentTime
    Set<StreamEvent> expectedEvents = Sets.newTreeSet(STREAM_EVENT_COMPARATOR);
    FileWriter<StreamEvent> writer = getFileWriterFactory().create(streamConfig, 0);
    try {
        expectedEvents.addAll(writeEvents(writer, "New event pre-flush ", 20, new IncrementingClock(approxEarliestNonExpiredTime, increment, 5)));
        writer.flush();
        expectedEvents.addAll(writeEvents(writer, "New event post-flush ", 20, new IncrementingClock(approxEarliestNonExpiredTime + 1, increment, 5)));
    } finally {
        writer.close();
    }
    StreamConsumer consumer = consumerFactory.create(streamId, stream, new ConsumerConfig(0L, 0, 1, DequeueStrategy.FIFO, null));
    verifyEvents(consumer, expectedEvents);
    TransactionContext txContext = createTxContext(consumer);
    txContext.start();
    try {
        // Should be no more pending events
        DequeueResult<StreamEvent> result = consumer.poll(1, 1, TimeUnit.SECONDS);
        Assert.assertTrue(result.isEmpty());
    } finally {
        txContext.finish();
    }
    consumer.close();
}
Also used : StreamId(co.cask.cdap.proto.id.StreamId) StreamEvent(co.cask.cdap.api.flow.flowlet.StreamEvent) Properties(java.util.Properties) TransactionContext(org.apache.tephra.TransactionContext) ConsumerConfig(co.cask.cdap.data2.queue.ConsumerConfig) Test(org.junit.Test)

Example 28 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class StreamConsumerTestBase method testFIFOReconfigure.

@Test
public void testFIFOReconfigure() throws Exception {
    String stream = "testReconfigure";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    streamAdmin.create(streamId);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    // Writes 5 events
    writeEvents(streamConfig, "Testing ", 5);
    // Configure 3 consumers.
    streamAdmin.configureInstances(streamId, 0L, 3);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    // Starts three consumers
    List<StreamConsumer> consumers = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        consumers.add(consumerFactory.create(streamId, "fifo.reconfigure", new ConsumerConfig(0L, i, 3, DequeueStrategy.FIFO, null)));
    }
    List<TransactionContext> txContexts = Lists.newArrayList();
    for (StreamConsumer consumer : consumers) {
        txContexts.add(createTxContext(consumer));
    }
    for (TransactionContext txContext : txContexts) {
        txContext.start();
    }
    // Consumer an item from each consumer, but only have the first one commit.
    for (int i = 0; i < consumers.size(); i++) {
        DequeueResult<StreamEvent> result = consumers.get(i).poll(1, 1, TimeUnit.SECONDS);
        Assert.assertEquals("Testing " + i, Charsets.UTF_8.decode(result.iterator().next().getBody()).toString());
        if (i == 0) {
            txContexts.get(i).finish();
        } else {
            txContexts.get(i).abort();
        }
    }
    for (StreamConsumer consumer : consumers) {
        consumer.close();
    }
    // Reconfigure to have two consumers.
    streamAdmin.configureInstances(streamId, 0L, 2);
    consumers.clear();
    for (int i = 0; i < 2; i++) {
        consumers.add(consumerFactory.create(streamId, "fifo.reconfigure", new ConsumerConfig(0L, i, 2, DequeueStrategy.FIFO, null)));
    }
    txContexts.clear();
    for (StreamConsumer consumer : consumers) {
        txContexts.add(createTxContext(consumer));
    }
    // Consumer an item from each consumer, they should see all four items.
    Set<String> messages = Sets.newTreeSet();
    boolean done;
    do {
        for (TransactionContext txContext : txContexts) {
            txContext.start();
        }
        done = true;
        for (int i = 0; i < consumers.size(); i++) {
            DequeueResult<StreamEvent> result = consumers.get(i).poll(1, 1, TimeUnit.SECONDS);
            if (result.isEmpty()) {
                continue;
            }
            done = false;
            messages.add(Charsets.UTF_8.decode(result.iterator().next().getBody()).toString());
            txContexts.get(i).finish();
        }
    } while (!done);
    Assert.assertEquals(4, messages.size());
    int count = 1;
    for (String msg : messages) {
        Assert.assertEquals("Testing " + count, msg);
        count++;
    }
    for (StreamConsumer consumer : consumers) {
        consumer.close();
    }
}
Also used : StreamId(co.cask.cdap.proto.id.StreamId) StreamEvent(co.cask.cdap.api.flow.flowlet.StreamEvent) TransactionContext(org.apache.tephra.TransactionContext) ConsumerConfig(co.cask.cdap.data2.queue.ConsumerConfig) Test(org.junit.Test)

Example 29 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class NamespaceHttpHandlerTest method testDeleteAll.

@Test
public void testDeleteAll() throws Exception {
    CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
    // test deleting non-existent namespace
    assertResponseCode(404, deleteNamespace("doesnotexist"));
    assertResponseCode(200, createNamespace(NAME));
    assertResponseCode(200, getNamespace(NAME));
    assertResponseCode(200, createNamespace(OTHER_NAME));
    assertResponseCode(200, getNamespace(OTHER_NAME));
    NamespacedLocationFactory namespacedLocationFactory = getInjector().getInstance(NamespacedLocationFactory.class);
    Location nsLocation = namespacedLocationFactory.get(new NamespaceId(NAME));
    Assert.assertTrue(nsLocation.exists());
    DatasetFramework dsFramework = getInjector().getInstance(DatasetFramework.class);
    StreamAdmin streamAdmin = getInjector().getInstance(StreamAdmin.class);
    deploy(AppWithServices.class, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
    deploy(AppWithDataset.class, Constants.Gateway.API_VERSION_3_TOKEN, NAME);
    deploy(AppWithStreamSizeSchedule.class, Constants.Gateway.API_VERSION_3_TOKEN, OTHER_NAME);
    deploy(AppForUnrecoverableResetTest.class, Constants.Gateway.API_VERSION_3_TOKEN, OTHER_NAME);
    DatasetId myDataset = new DatasetId(NAME, "myds");
    StreamId myStream = new StreamId(OTHER_NAME, "stream");
    Assert.assertTrue(dsFramework.hasInstance(myDataset));
    Assert.assertTrue(streamAdmin.exists(myStream));
    Id.Program program = Id.Program.from(NAME_ID, "AppWithServices", ProgramType.SERVICE, "NoOpService");
    startProgram(program);
    boolean resetEnabled = cConf.getBoolean(Constants.Dangerous.UNRECOVERABLE_RESET);
    cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, false);
    // because unrecoverable reset is disabled
    assertResponseCode(403, deleteNamespace(NAME));
    cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, resetEnabled);
    // because service is running
    assertResponseCode(409, deleteNamespace(NAME));
    Assert.assertTrue(nsLocation.exists());
    stopProgram(program);
    // delete should work now
    assertResponseCode(200, deleteNamespace(NAME));
    Assert.assertFalse(nsLocation.exists());
    Assert.assertFalse(dsFramework.hasInstance(myDataset));
    Assert.assertTrue(streamAdmin.exists(myStream));
    assertResponseCode(200, deleteNamespace(OTHER_NAME));
    Assert.assertFalse(streamAdmin.exists(myStream));
    // Create the namespace again and deploy the application containing schedules.
    // Application deployment should succeed.
    assertResponseCode(200, createNamespace(OTHER_NAME));
    HttpResponse response = deploy(AppForUnrecoverableResetTest.class, Constants.Gateway.API_VERSION_3_TOKEN, OTHER_NAME);
    Assert.assertEquals(200, response.getStatusLine().getStatusCode());
    assertResponseCode(200, deleteNamespace(OTHER_NAME));
}
Also used : DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) StreamAdmin(co.cask.cdap.data2.transaction.stream.StreamAdmin) StreamId(co.cask.cdap.proto.id.StreamId) HttpResponse(org.apache.http.HttpResponse) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) NamespaceId(co.cask.cdap.proto.id.NamespaceId) NamespaceId(co.cask.cdap.proto.id.NamespaceId) Id(co.cask.cdap.proto.Id) StreamId(co.cask.cdap.proto.id.StreamId) DatasetId(co.cask.cdap.proto.id.DatasetId) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Location(org.apache.twill.filesystem.Location) DatasetId(co.cask.cdap.proto.id.DatasetId) AppForUnrecoverableResetTest(co.cask.cdap.AppForUnrecoverableResetTest) Test(org.junit.Test)

Example 30 with StreamAdmin

use of co.cask.cdap.data2.transaction.stream.StreamAdmin in project cdap by caskdata.

the class LevelDBStreamConsumerTest method init.

@BeforeClass
public static void init() throws Exception {
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder().getAbsolutePath());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new NonCustomLocationUnitTestModule().getModule(), new SystemDatasetRuntimeModule().getInMemoryModules(), new DataSetsModules().getInMemoryModules(), new DataFabricLevelDBModule(), new TransactionMetricsModule(), new DiscoveryRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new ViewAdminModules().getInMemoryModules(), new NamespaceClientRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), Modules.override(new StreamAdminModules().getStandaloneModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            bind(StreamMetaStore.class).to(InMemoryStreamMetaStore.class);
            bind(NotificationFeedManager.class).to(NoOpNotificationFeedManager.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    }));
    consumerFactory = injector.getInstance(StreamConsumerFactory.class);
    streamAdmin = injector.getInstance(StreamAdmin.class);
    txClient = injector.getInstance(TransactionSystemClient.class);
    txManager = injector.getInstance(TransactionManager.class);
    queueClientFactory = injector.getInstance(QueueClientFactory.class);
    fileWriterFactory = injector.getInstance(StreamFileWriterFactory.class);
    streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
    streamCoordinatorClient.startAndWait();
    txManager.startAndWait();
    setupNamespaces(injector.getInstance(NamespacedLocationFactory.class));
}
Also used : NamespaceClientRuntimeModule(co.cask.cdap.common.namespace.guice.NamespaceClientRuntimeModule) ConfigModule(co.cask.cdap.common.guice.ConfigModule) UGIProvider(co.cask.cdap.security.impersonation.UGIProvider) UnsupportedUGIProvider(co.cask.cdap.security.impersonation.UnsupportedUGIProvider) NamespacedLocationFactory(co.cask.cdap.common.namespace.NamespacedLocationFactory) StreamConsumerFactory(co.cask.cdap.data2.transaction.stream.StreamConsumerFactory) DataFabricLevelDBModule(co.cask.cdap.data.runtime.DataFabricLevelDBModule) TransactionMetricsModule(co.cask.cdap.data.runtime.TransactionMetricsModule) ViewAdminModules(co.cask.cdap.data.view.ViewAdminModules) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) Injector(com.google.inject.Injector) InMemoryStreamMetaStore(co.cask.cdap.data.stream.service.InMemoryStreamMetaStore) StreamMetaStore(co.cask.cdap.data.stream.service.StreamMetaStore) SystemDatasetRuntimeModule(co.cask.cdap.data.runtime.SystemDatasetRuntimeModule) DiscoveryRuntimeModule(co.cask.cdap.common.guice.DiscoveryRuntimeModule) NotificationFeedManager(co.cask.cdap.notifications.feeds.NotificationFeedManager) NoOpNotificationFeedManager(co.cask.cdap.notifications.feeds.service.NoOpNotificationFeedManager) AuthenticationContextModules(co.cask.cdap.security.auth.context.AuthenticationContextModules) DataSetsModules(co.cask.cdap.data.runtime.DataSetsModules) DefaultOwnerAdmin(co.cask.cdap.security.impersonation.DefaultOwnerAdmin) OwnerAdmin(co.cask.cdap.security.impersonation.OwnerAdmin) StreamCoordinatorClient(co.cask.cdap.data.stream.StreamCoordinatorClient) NonCustomLocationUnitTestModule(co.cask.cdap.common.guice.NonCustomLocationUnitTestModule) AuthorizationTestModule(co.cask.cdap.security.authorization.AuthorizationTestModule) AbstractModule(com.google.inject.AbstractModule) StreamAdminModules(co.cask.cdap.data.stream.StreamAdminModules) StreamAdmin(co.cask.cdap.data2.transaction.stream.StreamAdmin) StreamFileWriterFactory(co.cask.cdap.data.stream.StreamFileWriterFactory) ExploreClientModule(co.cask.cdap.explore.guice.ExploreClientModule) TransactionManager(org.apache.tephra.TransactionManager) QueueClientFactory(co.cask.cdap.data2.queue.QueueClientFactory) AuthorizationEnforcementModule(co.cask.cdap.security.authorization.AuthorizationEnforcementModule) BeforeClass(org.junit.BeforeClass)

Aggregations

StreamAdmin (co.cask.cdap.data2.transaction.stream.StreamAdmin)25 StreamId (co.cask.cdap.proto.id.StreamId)18 Test (org.junit.Test)16 NamespacedLocationFactory (co.cask.cdap.common.namespace.NamespacedLocationFactory)14 Injector (com.google.inject.Injector)14 ConfigModule (co.cask.cdap.common.guice.ConfigModule)13 ExploreClientModule (co.cask.cdap.explore.guice.ExploreClientModule)13 AuthorizationEnforcementModule (co.cask.cdap.security.authorization.AuthorizationEnforcementModule)13 StreamEvent (co.cask.cdap.api.flow.flowlet.StreamEvent)12 DiscoveryRuntimeModule (co.cask.cdap.common.guice.DiscoveryRuntimeModule)12 DataSetsModules (co.cask.cdap.data.runtime.DataSetsModules)12 ViewAdminModules (co.cask.cdap.data.view.ViewAdminModules)12 AuthenticationContextModules (co.cask.cdap.security.auth.context.AuthenticationContextModules)12 AuthorizationTestModule (co.cask.cdap.security.authorization.AuthorizationTestModule)12 AbstractModule (com.google.inject.AbstractModule)12 BeforeClass (org.junit.BeforeClass)12 CConfiguration (co.cask.cdap.common.conf.CConfiguration)11 StreamMetaStore (co.cask.cdap.data.stream.service.StreamMetaStore)11 DefaultOwnerAdmin (co.cask.cdap.security.impersonation.DefaultOwnerAdmin)11 Configuration (org.apache.hadoop.conf.Configuration)11