Examples with StreamConsumerFactory co.cask.cdap.data2.transaction.stream.StreamConsumerFactory used on opensource projects

Example 6 with StreamConsumerFactory

use of co.cask.cdap.data2.transaction.stream.StreamConsumerFactory in project cdap by caskdata.

the class StreamConsumerTestBase method testFIFOReconfigure.

@Test
public void testFIFOReconfigure() throws Exception {
    String stream = "testReconfigure";
    StreamId streamId = TEST_NAMESPACE.stream(stream);
    StreamAdmin streamAdmin = getStreamAdmin();
    streamAdmin.create(streamId);
    StreamConfig streamConfig = streamAdmin.getConfig(streamId);
    // Writes 5 events
    writeEvents(streamConfig, "Testing ", 5);
    // Configure 3 consumers.
    streamAdmin.configureInstances(streamId, 0L, 3);
    StreamConsumerFactory consumerFactory = getConsumerFactory();
    // Starts three consumers
    List<StreamConsumer> consumers = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        consumers.add(consumerFactory.create(streamId, "fifo.reconfigure", new ConsumerConfig(0L, i, 3, DequeueStrategy.FIFO, null)));
    }
    List<TransactionContext> txContexts = Lists.newArrayList();
    for (StreamConsumer consumer : consumers) {
        txContexts.add(createTxContext(consumer));
    }
    for (TransactionContext txContext : txContexts) {
        txContext.start();
    }
    // Consumer an item from each consumer, but only have the first one commit.
    for (int i = 0; i < consumers.size(); i++) {
        DequeueResult<StreamEvent> result = consumers.get(i).poll(1, 1, TimeUnit.SECONDS);
        Assert.assertEquals("Testing " + i, Charsets.UTF_8.decode(result.iterator().next().getBody()).toString());
        if (i == 0) {
            txContexts.get(i).finish();
        } else {
            txContexts.get(i).abort();
        }
    }
    for (StreamConsumer consumer : consumers) {
        consumer.close();
    }
    // Reconfigure to have two consumers.
    streamAdmin.configureInstances(streamId, 0L, 2);
    consumers.clear();
    for (int i = 0; i < 2; i++) {
        consumers.add(consumerFactory.create(streamId, "fifo.reconfigure", new ConsumerConfig(0L, i, 2, DequeueStrategy.FIFO, null)));
    }
    txContexts.clear();
    for (StreamConsumer consumer : consumers) {
        txContexts.add(createTxContext(consumer));
    }
    // Consumer an item from each consumer, they should see all four items.
    Set<String> messages = Sets.newTreeSet();
    boolean done;
    do {
        for (TransactionContext txContext : txContexts) {
            txContext.start();
        }
        done = true;
        for (int i = 0; i < consumers.size(); i++) {
            DequeueResult<StreamEvent> result = consumers.get(i).poll(1, 1, TimeUnit.SECONDS);
            if (result.isEmpty()) {
                continue;
            }
            done = false;
            messages.add(Charsets.UTF_8.decode(result.iterator().next().getBody()).toString());
            txContexts.get(i).finish();
        }
    } while (!done);
    Assert.assertEquals(4, messages.size());
    int count = 1;
    for (String msg : messages) {
        Assert.assertEquals("Testing " + count, msg);
        count++;
    }
    for (StreamConsumer consumer : consumers) {
        consumer.close();
    }
}