Examples with PersistentTopic com.yahoo.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 76 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testActiveAndInActiveConsumerEntryCacheBehavior.

/**
     * Usecase 1: Only 1 Active Subscription - 1 subscriber - Produce Messages - EntryCache should cache messages -
     * EntryCache should be cleaned : Once active subscription consumes messages
     *
     * Usecase 2: 2 Active Subscriptions (faster and slower) and slower gets closed - 2 subscribers - Produce Messages -
     * 1 faster-subscriber consumes all messages and another slower-subscriber none - EntryCache should have cached
     * messages as slower-subscriber has not consumed messages yet - close slower-subscriber - EntryCache should be
     * cleared
     *
     * @throws Exception
     */
@Test
public void testActiveAndInActiveConsumerEntryCacheBehavior() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final long batchMessageDelayMs = 100;
    final int receiverSize = 10;
    final String topicName = "cache-topic";
    final String sub1 = "faster-sub1";
    final String sub2 = "slower-sub2";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(receiverSize);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    /************ usecase-1: *************/
    // 1. Subscriber Faster subscriber
    Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
    final String topic = "persistent://my-property/use/my-ns/" + topicName;
    Producer producer = pulsarClient.createProducer(topic, producerConf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(cacheField, cacheField.getModifiers() & ~Modifier.FINAL);
    EntryCacheImpl entryCache = spy((EntryCacheImpl) cacheField.get(ledger));
    cacheField.set(ledger, entryCache);
    Message msg = null;
    // 2. Produce messages
    for (int i = 0; i < 30; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // 3. Consume messages
    for (int i = 0; i < 30; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // Verify: EntryCache has been invalidated
    verify(entryCache, atLeastOnce()).invalidateEntries(any());
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    //
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: cache has to be cleared as there is no message needs to be consumed by active subscriber
    assertTrue(entryCache.getSize() == 0);
    /************ usecase-2: *************/
    // 1.b Subscriber slower-subscriber
    Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
    // Produce messages
    final int moreMessages = 10;
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // Consume messages
    for (int i = 0; i < receiverSize + moreMessages; i++) {
        msg = subscriber1.receive(5, TimeUnit.SECONDS);
        subscriber1.acknowledge(msg);
    }
    // sleep for a second: as ledger.updateCursorRateLimit RateLimiter will allow to invoke cursor-update after a
    // second
    //
    Thread.sleep(1000);
    // produce-consume one more message to trigger : ledger.internalReadFromLedger(..) which updates cursor and
    // EntryCache
    producer.send("message".getBytes());
    msg = subscriber1.receive(5, TimeUnit.SECONDS);
    // Verify: as active-subscriber2 has not consumed messages: EntryCache must have those entries in cache
    assertTrue(entryCache.getSize() != 0);
    // 3.b Close subscriber2: which will trigger cache to clear the cache
    subscriber2.close();
    // Verify: EntryCache should be cleared
    assertTrue(entryCache.getSize() == 0);
    subscriber1.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 77 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class ServerCnxTest method testProducerCommandWithAuthorizationPositive.

@Test(timeOut = 30000)
public void testProducerCommandWithAuthorizationPositive() throws Exception {
    AuthorizationManager authorizationManager = mock(AuthorizationManager.class);
    doReturn(CompletableFuture.completedFuture(true)).when(authorizationManager).canProduceAsync(Mockito.any(), Mockito.any());
    doReturn(authorizationManager).when(brokerService).getAuthorizationManager();
    doReturn(true).when(brokerService).isAuthenticationEnabled();
    resetChannel();
    setChannelConnected();
    // test PRODUCER success case
    ByteBuf clientCommand = Commands.newProducer(successTopicName, 1, /* producer id */
    1, /* request id */
    "prod-name");
    channel.writeInbound(clientCommand);
    assertEquals(getResponse().getClass(), CommandProducerSuccess.class);
    PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(successTopicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    channel.finish();
    assertEquals(topicRef.getProducers().size(), 0);
}

Example 78 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class ServerCnxTest method testSubscribeCommand.

@Test(timeOut = 30000)
public void testSubscribeCommand() throws Exception {
    final String failSubName = "failSub";
    resetChannel();
    setChannelConnected();
    doReturn(false).when(brokerService).isAuthenticationEnabled();
    doReturn(false).when(brokerService).isAuthorizationEnabled();
    // test SUBSCRIBE on topic and cursor creation success
    ByteBuf clientCommand = //
    Commands.newSubscribe(//
    successTopicName, successSubName, 1, /* consumer id */
    1, /* request id */
    SubType.Exclusive, 0, "test");
    channel.writeInbound(clientCommand);
    assertTrue(getResponse() instanceof CommandSuccess);
    PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(successTopicName);
    assertNotNull(topicRef);
    assertTrue(topicRef.getSubscriptions().containsKey(successSubName));
    assertTrue(topicRef.getPersistentSubscription(successSubName).getDispatcher().isConsumerConnected());
    // test SUBSCRIBE on topic creation success and cursor failure
    clientCommand = Commands.newSubscribe(successTopicName, failSubName, 2, 2, SubType.Exclusive, 0, "test");
    channel.writeInbound(clientCommand);
    assertTrue(getResponse() instanceof CommandError);
    // test SUBSCRIBE on topic creation failure
    clientCommand = Commands.newSubscribe(failTopicName, successSubName, 3, 3, SubType.Exclusive, 0, "test");
    channel.writeInbound(clientCommand);
    assertEquals(getResponse().getClass(), CommandError.class);
    // Server will not close the connection
    assertTrue(channel.isOpen());
    channel.finish();
}

Example 79 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class ServerCnxTest method testNonExistentTopicSuperUserAccess.

@Test(timeOut = 30000)
public void testNonExistentTopicSuperUserAccess() throws Exception {
    AuthorizationManager authorizationManager = spy(new AuthorizationManager(svcConfig, configCacheService));
    doReturn(authorizationManager).when(brokerService).getAuthorizationManager();
    doReturn(true).when(brokerService).isAuthorizationEnabled();
    doReturn(true).when(authorizationManager).isSuperUser(Mockito.anyString());
    // Test producer creation
    resetChannel();
    setChannelConnected();
    ByteBuf newProducerCmd = Commands.newProducer(nonExistentTopicName, 1, /* producer id */
    1, /* request id */
    "prod-name");
    channel.writeInbound(newProducerCmd);
    assertTrue(getResponse() instanceof CommandProducerSuccess);
    PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(nonExistentTopicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    channel.finish();
    // Test consumer creation
    resetChannel();
    setChannelConnected();
    ByteBuf newSubscribeCmd = //
    Commands.newSubscribe(//
    nonExistentTopicName, successSubName, 1, /* consumer id */
    1, /* request id */
    SubType.Exclusive, 0, "test");
    channel.writeInbound(newSubscribeCmd);
    topicRef = (PersistentTopic) brokerService.getTopicReference(nonExistentTopicName);
    assertNotNull(topicRef);
    assertTrue(topicRef.getSubscriptions().containsKey(successSubName));
    assertTrue(topicRef.getPersistentSubscription(successSubName).getDispatcher().isConsumerConnected());
    assertTrue(getResponse() instanceof CommandSuccess);
}

Example 80 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class ResendRequestTest method testFailoverSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testFailoverSingleAckedNormalTopic() throws Exception {
    String key = "testFailoverSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(10);
    conf.setSubscriptionType(SubscriptionType.Failover);
    conf.setConsumerName("consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    conf.setConsumerName("consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message message1;
    Message message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
    } while (message1 != null || message2 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 5. Consumer 1 asks for resend
    consumer1.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    // 6. Consumer 1 acknowledges a few messages
    receivedConsumer1 = receivedConsumer2 = 0;
    for (int i = 0; i < totalMessages / 2; i++) {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            log.info("Consumer 1 Acknowledged: " + new String(message1.getData()));
            consumer1.acknowledge(message1);
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
    }
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages / 2);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 7. Consumer 1 close
    consumer1.close();
    // 8. Checking if all messages are received by Consumer 2
    message2 = consumer2.receive();
    int acknowledgedMessages = 0;
    int unAcknowledgedMessages = 0;
    boolean flag = true;
    do {
        if (flag) {
            consumer2.acknowledge(message2);
            acknowledgedMessages += 1;
            log.info("Consumer 2 Acknowledged: " + new String(message2.getData()));
        } else {
            unAcknowledgedMessages += 1;
        }
        flag = !flag;
        log.info("Consumer 2 Received: " + new String(message2.getData()));
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message2 != null);
    log.info("Consumer 2 receives = " + (unAcknowledgedMessages + acknowledgedMessages));
    log.info("Consumer 2 acknowledges = " + acknowledgedMessages);
    assertEquals(unAcknowledgedMessages + acknowledgedMessages, totalMessages - receivedConsumer1);
    // 9 .Consumer 2 asks for a resend
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    message2 = consumer2.receive();
    receivedConsumer2 = 0;
    do {
        receivedConsumer2 += 1;
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message2 != null);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    assertEquals(unAcknowledgedMessages, receivedConsumer2);
}