Examples with ManagedLedgerImpl org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl used on opensource projects

Example 26 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class BrokerBkEnsemblesTests method testCrashBrokerWithoutCursorLedgerLeak.

/**
 * It verifies that broker deletes cursor-ledger when broker-crashes without closing topic gracefully
 *
 * <pre>
 * 1. Create topic : publish/consume-ack msgs to update new cursor-ledger
 * 2. Verify cursor-ledger is created and ledger-znode present
 * 3. Broker crashes: remove topic and managed-ledgers without closing
 * 4. Recreate topic: publish/consume-ack msgs to update new cursor-ledger
 * 5. Topic is recovered from old-ledger and broker deletes the old ledger
 * 6. verify znode of old-ledger is deleted
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testCrashBrokerWithoutCursorLedgerLeak() throws Exception {
    ZooKeeper zk = bkEnsemble.getZkClient();
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final String ns1 = "prop/usc/crash-broker";
    admin.namespaces().createNamespace(ns1);
    final String topic1 = "persistent://" + ns1 + "/my-topic";
    // (1) create topic
    // publish and ack messages so, cursor can create cursor-ledger and update metadata
    Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = client.newProducer().topic(topic1).create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        consumer.acknowledge(msg);
    }
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topic1).get();
    ManagedCursorImpl cursor = (ManagedCursorImpl) topic.getManagedLedger().getCursors().iterator().next();
    retryStrategically((test) -> cursor.getState().equals("Open"), 5, 100);
    // (2) validate cursor ledger is created and znode is present
    long cursorLedgerId = cursor.getCursorLedger();
    String ledgerPath = "/ledgers" + StringUtils.getHybridHierarchicalLedgerPath(cursorLedgerId);
    Assert.assertNotNull(zk.exists(ledgerPath, false));
    // (3) remove topic and managed-ledger from broker which means topic is not closed gracefully
    consumer.close();
    producer.close();
    pulsar.getBrokerService().removeTopicFromCache(topic1);
    ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory();
    Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
    field.setAccessible(true);
    @SuppressWarnings("unchecked") ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) field.get(factory);
    ledgers.clear();
    // (4) Recreate topic
    // publish and ack messages so, cursor can create cursor-ledger and update metadata
    consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    producer = client.newProducer().topic(topic1).create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        consumer.acknowledge(msg);
    }
    // (5) Broker should create new cursor-ledger and remove old cursor-ledger
    topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topic1).get();
    final ManagedCursorImpl cursor1 = (ManagedCursorImpl) topic.getManagedLedger().getCursors().iterator().next();
    retryStrategically((test) -> cursor1.getState().equals("Open"), 5, 100);
    long newCursorLedgerId = cursor1.getCursorLedger();
    Assert.assertNotEquals(newCursorLedgerId, -1);
    Assert.assertNotEquals(cursorLedgerId, newCursorLedgerId);
    // cursor node must be deleted
    Assert.assertNull(zk.exists(ledgerPath, false));
    producer.close();
    consumer.close();
    client.close();
}

Example 27 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class BrokerServiceTest method testLedgerOpenFailureShouldNotHaveDeadLock.

@Test
public void testLedgerOpenFailureShouldNotHaveDeadLock() throws Exception {
    final String namespace = "prop/usw/my-ns";
    final String deadLockTestTopic = "persistent://" + namespace + "/deadLockTestTopic";
    // let this broker own this namespace bundle by creating a topic
    try {
        final String successfulTopic = "persistent://" + namespace + "/ownBundleTopic";
        Producer<byte[]> producer = pulsarClient.newProducer().topic(successfulTopic).create();
        producer.close();
    } catch (Exception e) {
        fail(e.getMessage());
    }
    ExecutorService executor = Executors.newSingleThreadExecutor();
    BrokerService service = spy(pulsar.getBrokerService());
    // create topic will fail to get managedLedgerConfig
    CompletableFuture<ManagedLedgerConfig> failedManagedLedgerConfig = new CompletableFuture<>();
    failedManagedLedgerConfig.complete(null);
    doReturn(failedManagedLedgerConfig).when(service).getManagedLedgerConfig(anyObject());
    CompletableFuture<Void> topicCreation = new CompletableFuture<Void>();
    // fail managed-ledger future
    Field ledgerField = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
    ledgerField.setAccessible(true);
    @SuppressWarnings("unchecked") ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) ledgerField.get(pulsar.getManagedLedgerFactory());
    CompletableFuture<ManagedLedgerImpl> future = new CompletableFuture<>();
    future.completeExceptionally(new ManagedLedgerException("ledger opening failed"));
    ledgers.put(namespace + "/persistent/deadLockTestTopic", future);
    // create topic async and wait on the future completion
    executor.submit(() -> {
        service.getTopic(deadLockTestTopic).thenAccept(topic -> topicCreation.complete(null)).exceptionally(e -> {
            topicCreation.completeExceptionally(e.getCause());
            return null;
        });
    });
    // future-result should be completed with exception
    try {
        topicCreation.get(1, TimeUnit.SECONDS);
    } catch (TimeoutException | InterruptedException e) {
        fail("there is a dead-lock and it should have been prevented");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof PersistenceException);
    } finally {
        executor.shutdownNow();
        ledgers.clear();
    }
}

Example 28 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testGlobalNamespaceThrottling.

/**
 * <pre>
 * Verifies setting dispatch-rate on global namespace.
 * 1. It sets dispatch-rate for a local cluster into global-zk.policies
 * 2. Topic fetches dispatch-rate for the local cluster from policies
 * 3. applies dispatch rate
 *
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testGlobalNamespaceThrottling() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/global/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    final int messageRate = 5;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 360);
    admin.clusters().createCluster("global", new ClusterData("http://global:8080"));
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setNamespaceReplicationClusters(namespace, Lists.newArrayList("use"));
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0 || topic.getDispatchRateLimiter().getDispatchRateOnByte() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    int numMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        producer.send(new byte[80]);
    }
    // it can make sure that consumer had enough time to consume message but couldn't consume due to throttling
    Thread.sleep(500);
    // consumer should not have received all published message due to message-rate throttling
    Assert.assertNotEquals(totalReceived.get(), numMessages);
    consumer.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 29 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testClusterRateLimitingConfiguration.

@Test(dataProvider = "subscriptions", timeOut = 5000)
public void testClusterRateLimitingConfiguration(SubscriptionType subscription) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingBlock";
    final int messageRate = 5;
    int initValue = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg();
    // (1) Update message-dispatch-rate limit
    admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", Integer.toString(messageRate));
    // sleep incrementally as zk-watch notification is async and may take some time
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg() == initValue) {
            Thread.sleep(50 + (i * 10));
        }
    }
    Assert.assertNotEquals(pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg(), initValue);
    admin.namespaces().createNamespace(namespace);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    int numMessages = 500;
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        final String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // it can make sure that consumer had enough time to consume message but couldn't consume due to throttling
    Thread.sleep(500);
    // consumer should not have received all published message due to message-rate throttling
    Assert.assertNotEquals(totalReceived.get(), numMessages);
    consumer.close();
    producer.close();
    pulsar.getConfiguration().setDispatchThrottlingRatePerTopicInMsg(initValue);
    log.info("-- Exiting {} test --", methodName);
}

Example 30 with ManagedLedgerImpl

use of org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl in project incubator-pulsar by apache.

the class MessageDispatchThrottlingTest method testMessageRateLimitingReceiveAllMessagesAfterThrottling.

/**
 * verify rate-limiting should throttle message-dispatching based on message-rate
 *
 * <pre>
 *  1. dispatch-msg-rate = 10 msg/sec
 *  2. send 20 msgs
 *  3. it should take up to 2 second to receive all messages
 * </pre>
 *
 * @param subscription
 * @throws Exception
 */
@Test(dataProvider = "subscriptions", timeOut = 5000)
public void testMessageRateLimitingReceiveAllMessagesAfterThrottling(SubscriptionType subscription) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String namespace = "my-property/use/throttling_ns";
    final String topicName = "persistent://" + namespace + "/throttlingAll";
    final int messageRate = 10;
    DispatchRate dispatchRate = new DispatchRate(messageRate, -1, 1);
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setDispatchRate(namespace, dispatchRate);
    // create producer and topic
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    boolean isMessageRateUpdate = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getDispatchRateLimiter().getDispatchRateOnMsg() > 0) {
            isMessageRateUpdate = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(isMessageRateUpdate);
    Assert.assertEquals(admin.namespaces().getDispatchRate(namespace), dispatchRate);
    final int numProducedMessages = 20;
    final CountDownLatch latch = new CountDownLatch(numProducedMessages);
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name").subscriptionType(subscription).messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
        latch.countDown();
    }).subscribe();
    // deactive cursors
    deactiveCursors((ManagedLedgerImpl) topic.getManagedLedger());
    // Asynchronously produce messages
    for (int i = 0; i < numProducedMessages; i++) {
        final String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    latch.await();
    Assert.assertEquals(totalReceived.get(), numProducedMessages);
    consumer.close();
    producer.close();
    log.info("-- Exiting {} test --", methodName);
}