Examples with PersistentTopic org.apache.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 66 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class BrokerServiceTest method testBrokerServicePersistentTopicStats.

@Test
public void testBrokerServicePersistentTopicStats() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/successTopic";
    final String subName = "successSub";
    PersistentTopicStats stats;
    SubscriptionStats subStats;
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // subscription stats
    assertEquals(stats.subscriptions.keySet().size(), 1);
    assertEquals(subStats.msgBacklog, 0);
    assertEquals(subStats.consumers.size(), 1);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // publisher stats
    assertEquals(subStats.msgBacklog, 10);
    assertEquals(stats.publishers.size(), 1);
    assertTrue(stats.publishers.get(0).msgRateIn > 0.0);
    assertTrue(stats.publishers.get(0).msgThroughputIn > 0.0);
    assertTrue(stats.publishers.get(0).averageMsgSize > 0.0);
    assertNotNull(stats.publishers.get(0).clientVersion);
    // aggregated publish stats
    assertEquals(stats.msgRateIn, stats.publishers.get(0).msgRateIn);
    assertEquals(stats.msgThroughputIn, stats.publishers.get(0).msgThroughputIn);
    double diff = stats.averageMsgSize - stats.publishers.get(0).averageMsgSize;
    assertTrue(Math.abs(diff) < 0.000001);
    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    // aggregated consumer stats
    assertEquals(subStats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(subStats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(stats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertNotNull(subStats.consumers.get(0).clientVersion);
    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    assertEquals(subStats.msgBacklog, 0);
}

Example 67 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class AdminApiTest2 method testUpdatePersistencePolicyUpdateManagedCursor.

/**
 * validates update of persistent-policies reflects on managed-ledger and managed-cursor
 *
 * @throws Exception
 */
@Test
public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception {
    final String namespace = "prop-xyz/use/ns2";
    final String topicName = "persistent://" + namespace + "/topic1";
    admin.namespaces().createNamespace(namespace);
    admin.namespaces().setPersistence(namespace, new PersistencePolicies(3, 3, 3, 50.0));
    assertEquals(admin.namespaces().getPersistence(namespace), new PersistencePolicies(3, 3, 3, 50.0));
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topicName).get();
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topic.getManagedLedger();
    ManagedCursorImpl cursor = (ManagedCursorImpl) managedLedger.getCursors().iterator().next();
    final double newThrottleRate = 100;
    final int newEnsembleSize = 5;
    admin.namespaces().setPersistence(namespace, new PersistencePolicies(newEnsembleSize, 3, 3, newThrottleRate));
    retryStrategically((test) -> managedLedger.getConfig().getEnsembleSize() == newEnsembleSize && cursor.getThrottleMarkDelete() != newThrottleRate, 5, 200);
    // (1) verify cursor.markDelete has been updated
    assertEquals(cursor.getThrottleMarkDelete(), newThrottleRate);
    // (2) verify new ledger creation takes new config
    producer.close();
    consumer.close();
}

Example 68 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class AdminApiTest2 method testUnloadTopic.

/**
 * Verify unloading topic
 *
 * @throws Exception
 */
@Test(dataProvider = "topicType")
public void testUnloadTopic(final String topicType) throws Exception {
    final String namespace = "prop-xyz/use/ns2";
    final String topicName = topicType + "://" + namespace + "/topic1";
    admin.namespaces().createNamespace(namespace);
    // create a topic by creating a producer
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    producer.close();
    Topic topic = pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topic);
    final boolean isPersistentTopic = topic instanceof PersistentTopic;
    // (1) unload the topic
    unloadTopic(topicName, isPersistentTopic);
    topic = pulsar.getBrokerService().getTopicReference(topicName);
    // topic must be removed
    assertNull(topic);
    // recreation of producer will load the topic again
    producer = pulsarClient.newProducer().topic(topicName).create();
    topic = pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topic);
    // unload the topic
    unloadTopic(topicName, isPersistentTopic);
    // producer will retry and recreate the topic
    for (int i = 0; i < 5; i++) {
        topic = pulsar.getBrokerService().getTopicReference(topicName);
        if (topic == null || i != 4) {
            Thread.sleep(200);
        }
    }
    // topic should be loaded by this time
    topic = pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topic);
}

Example 69 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class NamespacesBase method clearBacklog.

private void clearBacklog(NamespaceName nsName, String bundleRange, String subscription) {
    try {
        List<Topic> topicList = pulsar().getBrokerService().getAllTopicsFromNamespaceBundle(nsName.toString(), nsName.toString() + "/" + bundleRange);
        List<CompletableFuture<Void>> futures = Lists.newArrayList();
        if (subscription != null) {
            if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) {
                subscription = PersistentReplicator.getRemoteCluster(subscription);
            }
            for (Topic topic : topicList) {
                if (topic instanceof PersistentTopic) {
                    futures.add(((PersistentTopic) topic).clearBacklog(subscription));
                }
            }
        } else {
            for (Topic topic : topicList) {
                if (topic instanceof PersistentTopic) {
                    futures.add(((PersistentTopic) topic).clearBacklog());
                }
            }
        }
        FutureUtil.waitForAll(futures).get();
    } catch (Exception e) {
        log.error("[{}] Failed to clear backlog for namespace {}/{}, subscription: {}", clientAppId(), nsName.toString(), bundleRange, subscription, e);
        throw new RestException(e);
    }
}

Example 70 with PersistentTopic

use of org.apache.pulsar.broker.service.persistent.PersistentTopic in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalResetCursor.

protected void internalResetCursor(String subName, long timestamp, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        int numParts = partitionMetadata.partitions;
        int numPartException = 0;
        Exception partitionException = null;
        try {
            for (int i = 0; i < numParts; i++) {
                pulsar().getAdminClient().persistentTopics().resetCursor(topicName.getPartition(i).toString(), subName, timestamp);
            }
        } catch (PreconditionFailedException pfe) {
            // throw the last exception if all partitions get this error
            // any other exception on partition is reported back to user
            ++numPartException;
            partitionException = pfe;
        } catch (Exception e) {
            log.warn("[{}] [{}] Failed to reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp, e);
            throw new RestException(e);
        }
        // report an error to user if unable to reset for all partitions
        if (numPartException == numParts) {
            log.warn("[{}] [{}] Failed to reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp, partitionException);
            throw new RestException(Status.PRECONDITION_FAILED, partitionException.getMessage());
        } else if (numPartException > 0) {
            log.warn("[{}][{}] partial errors for reset cursor on subscription {} to time {} - ", clientAppId(), topicName, subName, timestamp, partitionException);
        }
    } else {
        validateAdminOperationOnTopic(authoritative);
        log.info("[{}][{}] received reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp);
        PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
        if (topic == null) {
            throw new RestException(Status.NOT_FOUND, "Topic not found");
        }
        try {
            PersistentSubscription sub = topic.getSubscription(subName);
            checkNotNull(sub);
            sub.resetCursor(timestamp).get();
            log.info("[{}][{}] reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp);
        } catch (Exception e) {
            Throwable t = e.getCause();
            log.warn("[{}] [{}] Failed to reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp, e);
            if (e instanceof NullPointerException) {
                throw new RestException(Status.NOT_FOUND, "Subscription not found");
            } else if (e instanceof NotAllowedException) {
                throw new RestException(Status.METHOD_NOT_ALLOWED, e.getMessage());
            } else if (t instanceof SubscriptionInvalidCursorPosition) {
                throw new RestException(Status.PRECONDITION_FAILED, "Unable to find position for timestamp specified -" + t.getMessage());
            } else {
                throw new RestException(e);
            }
        }
    }
}