Examples with PersistentOfflineTopicStats org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats used on opensource projects

Example 1 with PersistentOfflineTopicStats

use of org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats in project incubator-pulsar by apache.

the class PersistentOfflineTopicStatsTest method testPersistentOfflineTopicStats.

@Test
public void testPersistentOfflineTopicStats() {
    PersistentOfflineTopicStats pot = new PersistentOfflineTopicStats("topic1", "prod1-broker1.messaging.use.example.com");
    String cursor = "cursor0";
    long time = System.currentTimeMillis();
    pot.addCursorDetails(cursor, 0, 1);
    pot.addLedgerDetails(0, time, 100, 1);
    Assert.assertEquals(pot.cursorDetails.get(cursor).cursorBacklog, 0);
    Assert.assertEquals(pot.cursorDetails.get(cursor).cursorLedgerId, 1);
    Assert.assertEquals(pot.dataLedgerDetails.get(0).entries, 0);
    Assert.assertEquals(pot.dataLedgerDetails.get(0).timestamp, time);
    Assert.assertEquals(pot.dataLedgerDetails.get(0).size, 100);
    Assert.assertEquals(pot.dataLedgerDetails.get(0).ledgerId, 1);
    long resetAt = System.currentTimeMillis();
    pot.reset();
    Assert.assertEquals(pot.storageSize, 0);
    Assert.assertEquals(pot.totalMessages, 0);
    Assert.assertEquals(pot.messageBacklog, 0);
    Assert.assertEquals(pot.dataLedgerDetails.size(), 0);
    Assert.assertEquals(pot.cursorDetails.size(), 0);
    Assert.assertTrue(pot.statGeneratedAt.getTime() - resetAt < 100);
}

Example 2 with PersistentOfflineTopicStats

use of org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalGetBacklog.

protected PersistentOfflineTopicStats internalGetBacklog(boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    // note that we do not want to load the topic and hence skip validateAdminOperationOnTopic()
    try {
        policiesCache().get(path(POLICIES, namespaceName.toString()));
    } catch (KeeperException.NoNodeException e) {
        log.warn("[{}] Failed to get topic backlog {}: Namespace does not exist", clientAppId(), namespaceName);
        throw new RestException(Status.NOT_FOUND, "Namespace does not exist");
    } catch (Exception e) {
        log.error("[{}] Failed to get topic backlog {}", clientAppId(), namespaceName, e);
        throw new RestException(e);
    }
    PersistentOfflineTopicStats offlineTopicStats = null;
    try {
        offlineTopicStats = pulsar().getBrokerService().getOfflineTopicStat(topicName);
        if (offlineTopicStats != null) {
            // offline topic stat has a cost - so use cached value until TTL
            long elapsedMs = System.currentTimeMillis() - offlineTopicStats.statGeneratedAt.getTime();
            if (TimeUnit.MINUTES.convert(elapsedMs, TimeUnit.MILLISECONDS) < OFFLINE_TOPIC_STAT_TTL_MINS) {
                return offlineTopicStats;
            }
        }
        final ManagedLedgerConfig config = pulsar().getBrokerService().getManagedLedgerConfig(topicName).get();
        ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(config.getDigestType(), config.getPassword(), pulsar().getAdvertisedAddress(), false);
        offlineTopicStats = offlineTopicBacklog.estimateUnloadedTopicBacklog((ManagedLedgerFactoryImpl) pulsar().getManagedLedgerFactory(), topicName);
        pulsar().getBrokerService().cacheOfflineTopicStats(topicName, offlineTopicStats);
    } catch (Exception exception) {
        throw new RestException(exception);
    }
    return offlineTopicStats;
}

Example 3 with PersistentOfflineTopicStats

use of org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats in project incubator-pulsar by apache.

the class ManagedLedgerOfflineBacklog method estimateUnloadedTopicBacklog.

public PersistentOfflineTopicStats estimateUnloadedTopicBacklog(ManagedLedgerFactoryImpl factory, TopicName topicName) throws Exception {
    String managedLedgerName = topicName.getPersistenceNamingEncoding();
    long numberOfEntries = 0;
    long totalSize = 0;
    final NavigableMap<Long, MLDataFormats.ManagedLedgerInfo.LedgerInfo> ledgers = new ConcurrentSkipListMap<>();
    final PersistentOfflineTopicStats offlineTopicStats = new PersistentOfflineTopicStats(managedLedgerName, brokerName);
    // calculate total managed ledger size and number of entries without loading the topic
    readLedgerMeta(factory, topicName, ledgers);
    for (MLDataFormats.ManagedLedgerInfo.LedgerInfo ls : ledgers.values()) {
        numberOfEntries += ls.getEntries();
        totalSize += ls.getSize();
        if (accurate) {
            offlineTopicStats.addLedgerDetails(ls.getEntries(), ls.getTimestamp(), ls.getSize(), ls.getLedgerId());
        }
    }
    offlineTopicStats.totalMessages = numberOfEntries;
    offlineTopicStats.storageSize = totalSize;
    if (log.isDebugEnabled()) {
        log.debug("[{}] Total number of entries - {} and size - {}", managedLedgerName, numberOfEntries, totalSize);
    }
    // calculate per cursor message backlog
    calculateCursorBacklogs(factory, topicName, ledgers, offlineTopicStats);
    offlineTopicStats.statGeneratedAt.setTime(System.currentTimeMillis());
    return offlineTopicStats;
}

Example 4 with PersistentOfflineTopicStats

use of org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats in project incubator-pulsar by apache.

the class ManagedLedgerBkTest method testOfflineTopicBacklog.

@Test
public void testOfflineTopicBacklog() throws Exception {
    ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();
    factoryConf.setMaxCacheSize(0);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc, factoryConf);
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
    ManagedLedger ledger = factory.open("property/cluster/namespace/my-ledger", config);
    ManagedCursor cursor = ledger.openCursor("c1");
    int N = 1;
    for (int i = 0; i < N; i++) {
        String entry = "entry-" + i;
        ledger.addEntry(entry.getBytes());
    }
    List<Entry> entries = cursor.readEntries(N);
    assertEquals(N, entries.size());
    entries.forEach(e -> e.release());
    ledger.close();
    ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(DigestType.CRC32, "".getBytes(Charsets.UTF_8), "", false);
    PersistentOfflineTopicStats offlineTopicStats = offlineTopicBacklog.getEstimatedUnloadedTopicBacklog((ManagedLedgerFactoryImpl) factory, "property/cluster/namespace/my-ledger");
    factory.shutdown();
    assertNotNull(offlineTopicStats);
}