Examples with TopicId co.cask.cdap.proto.id.TopicId used on opensource projects

Example 36 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class MessagingUtilsTest method testTopicConversion.

@Test
public void testTopicConversion() throws Exception {
    TopicId id = new TopicId("n1", "t1");
    byte[] topicBytes = MessagingUtils.toMetadataRowKey(id);
    TopicId topicId = MessagingUtils.toTopicId(topicBytes);
    Assert.assertEquals(id, topicId);
}

Example 37 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class LeaderElectionMessagingServiceTest method testFencing.

@Test
public void testFencing() throws IOException, InterruptedException, ExecutionException, TimeoutException {
    final TopicId topicId = NamespaceId.SYSTEM.topic("topic");
    // Change the fencing time
    long oldFencingDelay = cConf.getLong(Constants.MessagingSystem.HA_FENCING_DELAY_SECONDS);
    cConf.setLong(Constants.MessagingSystem.HA_FENCING_DELAY_SECONDS, 3L);
    try {
        Injector injector = createInjector(0);
        ZKClientService zkClient = injector.getInstance(ZKClientService.class);
        zkClient.startAndWait();
        final MessagingService messagingService = injector.getInstance(MessagingService.class);
        if (messagingService instanceof Service) {
            ((Service) messagingService).startAndWait();
        }
        // Shouldn't be serving request yet.
        try {
            messagingService.listTopics(NamespaceId.SYSTEM);
            Assert.fail("Expected service unavailable exception");
        } catch (ServiceUnavailableException e) {
        // expected
        }
        // Retry until pass the fencing delay (with some buffer)
        Tasks.waitFor(topicId, new Callable<TopicId>() {

            @Override
            public TopicId call() throws Exception {
                try {
                    return messagingService.getTopic(topicId).getTopicId();
                } catch (ServiceUnavailableException e) {
                    return null;
                }
            }
        }, 10L, TimeUnit.SECONDS, 200, TimeUnit.MILLISECONDS);
        if (messagingService instanceof Service) {
            ((Service) messagingService).stopAndWait();
        }
        zkClient.stopAndWait();
    } finally {
        cConf.setLong(Constants.MessagingSystem.HA_FENCING_DELAY_SECONDS, oldFencingDelay);
    }
}

Example 38 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testMaxSequence.

@Test
public void testMaxSequence() throws IOException {
    // This test the case when a single StoreRequest has more than SEQUENCE_ID_LIMIT (65536) payload.
    // Expected entries beyond the max seqId will be rolled to the next timestamp with seqId reset to start from 0
    // Generate SEQUENCE_ID_LIMIT + 1 payloads
    int msgCount = StoreRequestWriter.SEQUENCE_ID_LIMIT + 1;
    List<String> payloads = new ArrayList<>(msgCount);
    for (int i = 0; i < msgCount; i++) {
        payloads.add(Integer.toString(i));
    }
    TopicId topicId = new NamespaceId("ns1").topic("t1");
    TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
    // Write the payloads
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
    writer.persist(new TestStoreRequest(topicId, payloads), metadata);
    List<RawMessage> messages = testWriter.getMessages().get(topicId);
    Assert.assertEquals(msgCount, messages.size());
    // The first SEQUENCE_ID_LIMIT messages should be with the same timestamp, with seqId from 0 to SEQUENCE_ID_LIMIT
    for (int i = 0; i < StoreRequestWriter.SEQUENCE_ID_LIMIT; i++) {
        MessageId id = new MessageId(messages.get(i).getId());
        Assert.assertEquals(0L, id.getPublishTimestamp());
        Assert.assertEquals((short) i, id.getSequenceId());
    }
    // The (SEQUENCE_ID_LIMIT + 1)th message should have a different timestamp and seqId = 0
    MessageId id = new MessageId(messages.get(msgCount - 1).getId());
    Assert.assertEquals(1L, id.getPublishTimestamp());
    Assert.assertEquals(0, id.getPayloadSequenceId());
}

Example 39 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testBasic.

@Test
public void testBasic() throws IOException {
    TopicId topicId1 = new NamespaceId("ns1").topic("t1");
    TopicId topicId2 = new NamespaceId("ns2").topic("t2");
    TopicMetadata metadata1 = new TopicMetadata(topicId1, new HashMap<String, String>(), 1);
    TopicMetadata metadata2 = new TopicMetadata(topicId2, new HashMap<String, String>(), 1);
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider());
    ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
    writer.persist(new TestStoreRequest(topicId1, Arrays.asList("1", "2", "3")), metadata1);
    // There should be 3 messages being written
    List<RawMessage> messages = testWriter.getMessages().get(topicId1);
    Assert.assertEquals(3, messages.size());
    // All messages should be written with timestamp 0
    List<String> payloads = new ArrayList<>();
    for (RawMessage message : messages) {
        Assert.assertEquals(0L, new MessageId(message.getId()).getPublishTimestamp());
        payloads.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(Arrays.asList("1", "2", "3"), payloads);
    // Write to another topic
    writer.persist(new TestStoreRequest(topicId2, Arrays.asList("a", "b", "c")), metadata2);
    // There should be 3 messages being written to topic2
    messages = testWriter.getMessages().get(topicId2);
    Assert.assertEquals(3, messages.size());
    // All messages should be written with timestamp 1
    payloads.clear();
    for (RawMessage message : messages) {
        Assert.assertEquals(1L, new MessageId(message.getId()).getPublishTimestamp());
        payloads.add(Bytes.toString(message.getPayload()));
    }
    Assert.assertEquals(Arrays.asList("a", "b", "c"), payloads);
}

Example 40 with TopicId

use of co.cask.cdap.proto.id.TopicId in project cdap by caskdata.

the class ConcurrentMessageWriterTest method testConcurrentWrites.

@Test
public void testConcurrentWrites() throws InterruptedException, BrokenBarrierException {
    int payloadsPerRequest = 200;
    int threadCount = 20;
    final int requestPerThread = 20;
    long writeLatencyMillis = 50L;
    final TopicId topicId = NamespaceId.DEFAULT.topic("t");
    final TopicMetadata metadata = new TopicMetadata(topicId, new HashMap<String, String>(), 1);
    TestStoreRequestWriter testWriter = new TestStoreRequestWriter(new TimeProvider.IncrementalTimeProvider(), writeLatencyMillis);
    final ConcurrentMessageWriter writer = new ConcurrentMessageWriter(testWriter);
    final List<String> payload = new ArrayList<>(payloadsPerRequest);
    for (int i = 0; i < payloadsPerRequest; i++) {
        payload.add(Integer.toString(i));
    }
    ExecutorService executor = Executors.newFixedThreadPool(threadCount);
    final CyclicBarrier barrier = new CyclicBarrier(threadCount + 1);
    for (int i = 0; i < threadCount; i++) {
        final int threadId = i;
        executor.submit(new Runnable() {

            @Override
            public void run() {
                Stopwatch stopwatch = new Stopwatch();
                try {
                    barrier.await();
                    stopwatch.start();
                    for (int i = 0; i < requestPerThread; i++) {
                        writer.persist(new TestStoreRequest(topicId, payload), metadata);
                    }
                    LOG.info("Complete time for thread {} is {} ms", threadId, stopwatch.elapsedMillis());
                } catch (Exception e) {
                    LOG.error("Exception raised when persisting.", e);
                }
            }
        });
    }
    Stopwatch stopwatch = new Stopwatch();
    barrier.await();
    stopwatch.start();
    executor.shutdown();
    Assert.assertTrue(executor.awaitTermination(1, TimeUnit.MINUTES));
    LOG.info("Total time passed: {} ms", stopwatch.elapsedMillis());
    // Validate that the total number of messages written is correct
    List<RawMessage> messages = testWriter.getMessages().get(topicId);
    Assert.assertEquals(payloadsPerRequest * threadCount * requestPerThread, messages.size());
    // The message id must be sorted
    RawMessage lastMessage = null;
    for (RawMessage message : messages) {
        if (lastMessage != null) {
            Assert.assertTrue(Bytes.compareTo(lastMessage.getId(), message.getId()) < 0);
        }
        lastMessage = message;
    }
}