Examples with PersistentTopic com.yahoo.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 31 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class ReplicatorTest method testResumptionAfterBacklogRelaxed.

/**
     * Issue #199
     *
     * It verifies that: if the remote cluster reaches backlog quota limit, replicator temporarily stops and once the
     * backlog drains it should resume replication.
     *
     * @throws Exception
     */
@Test(enabled = true, priority = -1)
public void testResumptionAfterBacklogRelaxed() throws Exception {
    List<RetentionPolicy> policies = Lists.newArrayList();
    policies.add(RetentionPolicy.producer_exception);
    policies.add(RetentionPolicy.producer_request_hold);
    for (RetentionPolicy policy : policies) {
        DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns1/%s", policy));
        // Producer on r1
        MessageProducer producer1 = new MessageProducer(url1, dest);
        // Consumer on r1
        MessageConsumer consumer1 = new MessageConsumer(url1, dest);
        // Consumer on r2
        MessageConsumer consumer2 = new MessageConsumer(url2, dest);
        // Replicator for r1 -> r2
        PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
        PersistentReplicator replicator = topic.getPersistentReplicator("r2");
        // Restrict backlog quota limit to 1
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1, policy));
        // Produce a message to r1, then it will be replicated to r2 and fulfill the backlog.
        producer1.produce(1);
        consumer1.receive(1);
        Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
        // Produce 9 messages to r1, then it will be pended because of the backlog limit excess
        producer1.produce(9);
        consumer1.receive(9);
        Thread.sleep(1000L);
        assertEquals(replicator.getStats().replicationBacklog, 9);
        // Relax backlog quota limit to 1G
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1024 * 1024 * 1024, policy));
        Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
        // The messages should be replicated to r2
        assertEquals(replicator.getStats().replicationBacklog, 0);
        consumer2.receive(1);
        consumer2.receive(9);
        if (!consumer2.drained()) {
            throw new Exception("consumer2 - unexpected message in queue");
        }
        producer1.close();
        consumer1.close();
        consumer2.close();
    }
}

Example 32 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class ReplicatorTest method testConcurrentReplicator.

@Test
public void testConcurrentReplicator() throws Exception {
    log.info("--- Starting ReplicatorTest::testConcurrentReplicator ---");
    final DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns1/topic-%d", 0));
    ClientConfiguration conf = new ClientConfiguration();
    conf.setStatsInterval(0, TimeUnit.SECONDS);
    Producer producer = PulsarClient.create(url1.toString(), conf).createProducer(dest.toString());
    producer.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(dest.toString()).get();
    PulsarClientImpl pulsarClient = spy((PulsarClientImpl) pulsar1.getBrokerService().getReplicationClient("r3"));
    final Method startRepl = PersistentTopic.class.getDeclaredMethod("startReplicator", String.class);
    startRepl.setAccessible(true);
    Field replClientField = BrokerService.class.getDeclaredField("replicationClients");
    replClientField.setAccessible(true);
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients = (ConcurrentOpenHashMap<String, PulsarClient>) replClientField.get(pulsar1.getBrokerService());
    replicationClients.put("r3", pulsarClient);
    ExecutorService executor = Executors.newFixedThreadPool(5);
    for (int i = 0; i < 5; i++) {
        executor.submit(() -> {
            try {
                startRepl.invoke(topic, "r3");
            } catch (Exception e) {
                fail("setting replicator failed", e);
            }
        });
    }
    Thread.sleep(3000);
    Mockito.verify(pulsarClient, Mockito.times(1)).createProducerAsync(Mockito.anyString(), Mockito.anyObject(), Mockito.anyString());
}

Example 33 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testDeactivatingBacklogConsumer.

@Test
public void testDeactivatingBacklogConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final long batchMessageDelayMs = 100;
    final int receiverSize = 10;
    final String topicName = "cache-topic";
    final String topic = "persistent://my-property/use/my-ns/" + topicName;
    final String sub1 = "faster-sub1";
    final String sub2 = "slower-sub2";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(receiverSize);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    // 1. Subscriber Faster subscriber: let it consume all messages immediately
    Consumer subscriber1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub1, conf);
    // 1.b. Subscriber Slow subscriber:
    conf.setReceiverQueueSize(receiverSize);
    Consumer subscriber2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/" + topicName, sub2, conf);
    Producer producer = pulsarClient.createProducer(topic, producerConf);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topic);
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    // reflection to set/get cache-backlog fields value:
    final long maxMessageCacheRetentionTimeMillis = 100;
    Field backlogThresholdField = ManagedLedgerImpl.class.getDeclaredField("maxActiveCursorBacklogEntries");
    backlogThresholdField.setAccessible(true);
    Field field = ManagedLedgerImpl.class.getDeclaredField("maxMessageCacheRetentionTimeMillis");
    field.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
    field.set(ledger, maxMessageCacheRetentionTimeMillis);
    final long maxActiveCursorBacklogEntries = (long) backlogThresholdField.get(ledger);
    Message msg = null;
    final int totalMsgs = (int) maxActiveCursorBacklogEntries + receiverSize + 1;
    // 2. Produce messages
    for (int i = 0; i < totalMsgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // 3. Consume messages: at Faster subscriber
    for (int i = 0; i < totalMsgs; i++) {
        msg = subscriber1.receive(100, TimeUnit.MILLISECONDS);
        subscriber1.acknowledge(msg);
    }
    // wait : so message can be eligible to to be evict from cache
    Thread.sleep(maxMessageCacheRetentionTimeMillis);
    // 4. deactivate subscriber which has built the backlog
    ledger.checkBackloggedCursors();
    Thread.sleep(100);
    // 5. verify: active subscribers
    Set<String> activeSubscriber = Sets.newHashSet();
    ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
    assertTrue(activeSubscriber.contains(sub1));
    assertFalse(activeSubscriber.contains(sub2));
    // 6. consume messages : at slower subscriber
    for (int i = 0; i < totalMsgs; i++) {
        msg = subscriber2.receive(100, TimeUnit.MILLISECONDS);
        subscriber2.acknowledge(msg);
    }
    ledger.checkBackloggedCursors();
    activeSubscriber.clear();
    ledger.getActiveCursors().forEach(c -> activeSubscriber.add(c.getName()));
    assertTrue(activeSubscriber.contains(sub1));
    assertTrue(activeSubscriber.contains(sub2));
}

Example 34 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class ServerCnxTest method testUnsupportedBatchMsgSubscribeCommand.

@Test(timeOut = 30000)
public void testUnsupportedBatchMsgSubscribeCommand() throws Exception {
    final String failSubName = "failSub";
    resetChannel();
    setChannelConnected();
    setConnectionVersion(ProtocolVersion.v3.getNumber());
    doReturn(false).when(brokerService).isAuthenticationEnabled();
    doReturn(false).when(brokerService).isAuthorizationEnabled();
    // test SUBSCRIBE on topic and cursor creation success
    ByteBuf clientCommand = //
    Commands.newSubscribe(//
    successTopicName, successSubName, 1, /* consumer id */
    1, /* request id */
    SubType.Exclusive, 0, /* priority */
    "test");
    channel.writeInbound(clientCommand);
    assertTrue(getResponse() instanceof CommandSuccess);
    PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(successTopicName);
    topicRef.markBatchMessagePublished();
    // test SUBSCRIBE on topic and cursor creation success
    clientCommand = Commands.newSubscribe(successTopicName, failSubName, 2, 2, SubType.Exclusive, 0, /* priority */
    "test");
    channel.writeInbound(clientCommand);
    Object response = getResponse();
    assertTrue(response instanceof CommandError);
    assertTrue(((CommandError) response).getError().equals(ServerError.UnsupportedVersionError));
    // Server will not close the connection
    assertTrue(channel.isOpen());
    channel.finish();
}

Example 35 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BrokerService method removeTopicFromCache.

public void removeTopicFromCache(String topic) {
    try {
        DestinationName destination = DestinationName.get(topic);
        NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getBundle(destination);
        checkArgument(namespaceBundle instanceof NamespaceBundle);
        String bundleName = namespaceBundle.toString();
        String namespaceName = destination.getNamespaceObject().toString();
        synchronized (multiLayerTopicsMap) {
            ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, PersistentTopic>> namespaceMap = multiLayerTopicsMap.get(namespaceName);
            ConcurrentOpenHashMap<String, PersistentTopic> bundleMap = namespaceMap.get(bundleName);
            bundleMap.remove(topic);
            if (bundleMap.isEmpty()) {
                namespaceMap.remove(bundleName);
            }
            if (namespaceMap.isEmpty()) {
                multiLayerTopicsMap.remove(namespaceName);
                final ClusterReplicationMetrics clusterReplicationMetrics = pulsarStats.getClusterReplicationMetrics();
                replicationClients.forEach((cluster, client) -> {
                    clusterReplicationMetrics.remove(clusterReplicationMetrics.getKeyName(namespaceName, cluster));
                });
            }
        }
    } catch (Exception e) {
        log.warn("Got exception when retrieving bundle name during removeTopicFromCache", e);
    }
    topics.remove(topic);
}