Examples with NonPersistentTopic org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic used on opensource projects

Example 6 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testReplicator.

/**
 * verifies that non-persistent topic replicates using replicator
 */
@Test
public void testReplicator() throws Exception {
    ReplicationClusterManager replication = new ReplicationClusterManager();
    replication.setupReplicationCluster();
    try {
        final String globalTopicName = "non-persistent://pulsar/global/ns/nonPersistentTopic";
        final int timeWaitToSync = 100;
        NonPersistentTopicStats stats;
        SubscriptionStats subStats;
        PulsarClient client1 = PulsarClient.builder().serviceUrl(replication.url1.toString()).build();
        PulsarClient client2 = PulsarClient.builder().serviceUrl(replication.url2.toString()).build();
        PulsarClient client3 = PulsarClient.builder().serviceUrl(replication.url3.toString()).build();
        ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName).subscriptionName("subscriber-2").subscribe();
        ConsumerImpl<byte[]> repl2Consumer = (ConsumerImpl<byte[]>) client2.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        ConsumerImpl<byte[]> repl3Consumer = (ConsumerImpl<byte[]>) client3.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        Producer<byte[]> producer = client1.newProducer().topic(globalTopicName).create();
        Thread.sleep(timeWaitToSync);
        PulsarService replicationPulasr = replication.pulsar1;
        // Replicator for r1 -> r2,r3
        NonPersistentTopic topicRef = (NonPersistentTopic) replication.pulsar1.getBrokerService().getTopicReference(globalTopicName);
        NonPersistentReplicator replicatorR2 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r2");
        NonPersistentReplicator replicatorR3 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r3");
        assertNotNull(topicRef);
        assertNotNull(replicatorR2);
        assertNotNull(replicatorR3);
        rolloverPerIntervalStats(replicationPulasr);
        stats = topicRef.getStats();
        subStats = stats.getSubscriptions().values().iterator().next();
        // subscription stats
        assertEquals(stats.getSubscriptions().keySet().size(), 2);
        assertEquals(subStats.consumers.size(), 1);
        Thread.sleep(timeWaitToSync);
        int totalProducedMessages = 100;
        for (int i = 0; i < totalProducedMessages; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (1) consume by consumer1
        Message<?> msg = null;
        Set<String> messageSet = Sets.newHashSet();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = consumer1.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (2) consume by consumer2
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = consumer2.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (3) consume by repl2consumer
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = repl2Consumer.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (4) consume by repl3consumer
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = repl3Consumer.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        Thread.sleep(timeWaitToSync);
        rolloverPerIntervalStats(replicationPulasr);
        stats = topicRef.getStats();
        subStats = stats.getSubscriptions().values().iterator().next();
        assertTrue(subStats.msgRateOut > 0);
        assertEquals(subStats.consumers.size(), 1);
        assertTrue(subStats.msgThroughputOut > 0);
        // consumer stats
        assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
        assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
        assertEquals(subStats.msgRateRedeliver, 0.0);
        producer.close();
        consumer1.close();
        repl2Consumer.close();
        repl3Consumer.close();
        client1.close();
        client2.close();
        client3.close();
    } finally {
        replication.shutdownReplicationCluster();
    }
}

Example 7 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testLoadManagerAssignmentForNonPersistentTestAssignment.

/**
 * verifies load manager assigns topic only if broker started in non-persistent mode
 *
 * <pre>
 * 1. Start broker with disable non-persistent topic mode
 * 2. Create namespace with non-persistency set
 * 3. Create non-persistent topic
 * 4. Load-manager should not be able to find broker
 * 5. Create producer on that topic should fail
 * </pre>
 */
@Test(dataProvider = "loadManager")
public void testLoadManagerAssignmentForNonPersistentTestAssignment(String loadManagerName) throws Exception {
    final String namespace = "my-property/use/my-ns";
    final String topicName = "non-persistent://" + namespace + "/loadManager";
    final String defaultLoadManagerName = conf.getLoadManagerClassName();
    final boolean defaultENableNonPersistentTopic = conf.isEnableNonPersistentTopics();
    try {
        // start broker to not own non-persistent namespace and create non-persistent namespace
        stopBroker();
        conf.setEnableNonPersistentTopics(false);
        conf.setLoadManagerClassName(loadManagerName);
        startBroker();
        Field field = PulsarService.class.getDeclaredField("loadManager");
        field.setAccessible(true);
        @SuppressWarnings("unchecked") AtomicReference<LoadManager> loadManagerRef = (AtomicReference<LoadManager>) field.get(pulsar);
        LoadManager manager = LoadManager.create(pulsar);
        manager.start();
        loadManagerRef.set(manager);
        NamespaceBundle fdqn = pulsar.getNamespaceService().getBundle(TopicName.get(topicName));
        LoadManager loadManager = pulsar.getLoadManager().get();
        ResourceUnit broker = null;
        try {
            broker = loadManager.getLeastLoaded(fdqn).get();
        } catch (Exception e) {
        // Ok. (ModulearLoadManagerImpl throws RuntimeException incase don't find broker)
        }
        assertNull(broker);
        try {
            Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).createAsync().get(1, TimeUnit.SECONDS);
            producer.close();
            fail("topic loading should have failed");
        } catch (Exception e) {
        // Ok
        }
        NonPersistentTopic topicRef = (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
        assertNull(topicRef);
    } finally {
        conf.setEnableNonPersistentTopics(defaultENableNonPersistentTopic);
        conf.setLoadManagerClassName(defaultLoadManagerName);
    }
}

Example 8 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testNonPersistentBrokerModeRejectPersistentTopic.

/**
 * verifies that broker started with onlyNonPersistent mode doesn't own persistent-topic
 *
 * @param loadManagerName
 * @throws Exception
 */
@Test(dataProvider = "loadManager")
public void testNonPersistentBrokerModeRejectPersistentTopic(String loadManagerName) throws Exception {
    final String namespace = "my-property/use/my-ns";
    final String topicName = "persistent://" + namespace + "/loadManager";
    final String defaultLoadManagerName = conf.getLoadManagerClassName();
    final boolean defaultEnablePersistentTopic = conf.isEnablePersistentTopics();
    final boolean defaultEnableNonPersistentTopic = conf.isEnableNonPersistentTopics();
    try {
        // start broker to not own non-persistent namespace and create non-persistent namespace
        stopBroker();
        conf.setEnableNonPersistentTopics(true);
        conf.setEnablePersistentTopics(false);
        conf.setLoadManagerClassName(loadManagerName);
        startBroker();
        Field field = PulsarService.class.getDeclaredField("loadManager");
        field.setAccessible(true);
        @SuppressWarnings("unchecked") AtomicReference<LoadManager> loadManagerRef = (AtomicReference<LoadManager>) field.get(pulsar);
        LoadManager manager = LoadManager.create(pulsar);
        manager.start();
        loadManagerRef.set(manager);
        NamespaceBundle fdqn = pulsar.getNamespaceService().getBundle(TopicName.get(topicName));
        LoadManager loadManager = pulsar.getLoadManager().get();
        ResourceUnit broker = null;
        try {
            broker = loadManager.getLeastLoaded(fdqn).get();
        } catch (Exception e) {
        // Ok. (ModulearLoadManagerImpl throws RuntimeException incase don't find broker)
        }
        assertNull(broker);
        try {
            Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).createAsync().get(1, TimeUnit.SECONDS);
            producer.close();
            fail("topic loading should have failed");
        } catch (Exception e) {
        // Ok
        }
        NonPersistentTopic topicRef = (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
        assertNull(topicRef);
    } finally {
        conf.setEnablePersistentTopics(defaultEnablePersistentTopic);
        conf.setEnableNonPersistentTopics(defaultEnableNonPersistentTopic);
        conf.setLoadManagerClassName(defaultLoadManagerName);
    }
}

Example 9 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testNonPersistentTopicUnderPersistentNamespace.

/**
 * verifies: broker should reject non-persistent topic loading if broker is not enable for non-persistent topic
 *
 * @param loadManagerName
 * @throws Exception
 */
@Test
public void testNonPersistentTopicUnderPersistentNamespace() throws Exception {
    final String namespace = "my-property/use/my-ns";
    final String topicName = "non-persistent://" + namespace + "/persitentNamespace";
    final boolean defaultENableNonPersistentTopic = conf.isEnableNonPersistentTopics();
    try {
        conf.setEnableNonPersistentTopics(false);
        stopBroker();
        startBroker();
        try {
            Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).createAsync().get(1, TimeUnit.SECONDS);
            producer.close();
            fail("topic loading should have failed");
        } catch (Exception e) {
        // Ok
        }
        NonPersistentTopic topicRef = (NonPersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
        assertNull(topicRef);
    } finally {
        conf.setEnableNonPersistentTopics(defaultENableNonPersistentTopic);
    }
}

Example 10 with NonPersistentTopic

use of org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic in project incubator-pulsar by apache.

the class BrokerService method createNonPersistentTopic.

private CompletableFuture<Topic> createNonPersistentTopic(String topic) {
    CompletableFuture<Topic> topicFuture = new CompletableFuture<Topic>();
    if (!pulsar.getConfiguration().isEnableNonPersistentTopics()) {
        if (log.isDebugEnabled()) {
            log.debug("Broker is unable to load non-persistent topic {}", topic);
        }
        topicFuture.completeExceptionally(new NotAllowedException("Broker is not unable to load non-persistent topic"));
        return topicFuture;
    }
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    NonPersistentTopic nonPersistentTopic = new NonPersistentTopic(topic, this);
    CompletableFuture<Void> replicationFuture = nonPersistentTopic.checkReplication();
    replicationFuture.thenRun(() -> {
        log.info("Created topic {}", nonPersistentTopic);
        long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
        pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
        addTopicToStatsMaps(TopicName.get(topic), nonPersistentTopic);
        topicFuture.complete(nonPersistentTopic);
    });
    replicationFuture.exceptionally((ex) -> {
        log.warn("Replication check failed. Removing topic from topics list {}, {}", topic, ex);
        nonPersistentTopic.stopReplProducers().whenComplete((v, exception) -> {
            pulsar.getExecutor().submit(() -> topics.remove(topic, topicFuture));
            topicFuture.completeExceptionally(ex);
        });
        return null;
    });
    return topicFuture;
}