Examples with ManagedLedgerConfig org.apache.bookkeeper.mledger.ManagedLedgerConfig used on opensource projects

Example 96 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class ManagedCursorTest method testNumberOfEntries.

@Test(timeOut = 20000)
void testNumberOfEntries() throws Exception {
    ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
    ManagedCursor c1 = ledger.openCursor("c1");
    ledger.addEntry("dummy-entry-1".getBytes(Encoding));
    ManagedCursor c2 = ledger.openCursor("c2");
    ledger.addEntry("dummy-entry-2".getBytes(Encoding));
    ManagedCursor c3 = ledger.openCursor("c3");
    ledger.addEntry("dummy-entry-3".getBytes(Encoding));
    ManagedCursor c4 = ledger.openCursor("c4");
    ledger.addEntry("dummy-entry-4".getBytes(Encoding));
    ManagedCursor c5 = ledger.openCursor("c5");
    assertEquals(c1.getNumberOfEntries(), 4);
    assertEquals(c1.hasMoreEntries(), true);
    assertEquals(c2.getNumberOfEntries(), 3);
    assertEquals(c2.hasMoreEntries(), true);
    assertEquals(c3.getNumberOfEntries(), 2);
    assertEquals(c3.hasMoreEntries(), true);
    assertEquals(c4.getNumberOfEntries(), 1);
    assertEquals(c4.hasMoreEntries(), true);
    assertEquals(c5.getNumberOfEntries(), 0);
    assertEquals(c5.hasMoreEntries(), false);
    List<Entry> entries = c1.readEntries(2);
    assertEquals(entries.size(), 2);
    c1.markDelete(entries.get(1).getPosition());
    assertEquals(c1.getNumberOfEntries(), 2);
    entries.forEach(e -> e.release());
}

Example 97 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class ManagedCursorTest method internalTestFindNewestMatchingAllEntries.

void internalTestFindNewestMatchingAllEntries(final String name, final int entriesPerLedger, final int expectedEntryId) throws Exception {
    final String ledgerAndCursorName = name;
    ManagedLedgerConfig config = new ManagedLedgerConfig();
    config.setRetentionSizeInMB(10);
    config.setMaxEntriesPerLedger(entriesPerLedger);
    config.setRetentionTime(1, TimeUnit.HOURS);
    ManagedLedger ledger = factory.open(ledgerAndCursorName, config);
    ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName);
    ledger.addEntry(getEntryPublishTime("retained1"));
    // space apart message publish times
    Thread.sleep(100);
    ledger.addEntry(getEntryPublishTime("retained2"));
    Thread.sleep(100);
    ledger.addEntry(getEntryPublishTime("retained3"));
    Thread.sleep(100);
    Position newPosition = ledger.addEntry(getEntryPublishTime("expectedresetposition"));
    long timestamp = System.currentTimeMillis();
    long ledgerId = ((PositionImpl) newPosition).getLedgerId();
    Thread.sleep(2);
    ledger.addEntry(getEntryPublishTime("not-read"));
    List<Entry> entries = c1.readEntries(3);
    c1.markDelete(entries.get(2).getPosition());
    c1.close();
    ledger.close();
    entries.forEach(e -> e.release());
    // give timed ledger trimming a chance to run
    Thread.sleep(100);
    ledger = factory.open(ledgerAndCursorName, config);
    c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName);
    PositionImpl found = (PositionImpl) findPositionFromAllEntries(c1, timestamp);
    assertEquals(found.getLedgerId(), ledgerId);
    assertEquals(found.getEntryId(), expectedEntryId);
}

Example 98 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class ManagedCursorTest method readWithCacheDisabled.

@Test(timeOut = 20000)
void readWithCacheDisabled() throws Exception {
    ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
    config.setMaxCacheSize(0);
    factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
    ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
    ManagedCursor c1 = ledger.openCursor("c1");
    ManagedCursor c2 = ledger.openCursor("c2");
    ledger.addEntry("entry-1".getBytes(Encoding));
    ledger.addEntry("entry-2".getBytes(Encoding));
    List<Entry> entries = c1.readEntries(2);
    assertEquals(entries.size(), 2);
    assertEquals(new String(entries.get(0).getData(), Encoding), "entry-1");
    assertEquals(new String(entries.get(1).getData(), Encoding), "entry-2");
    entries.forEach(e -> e.release());
    entries = c1.readEntries(2);
    assertEquals(entries.size(), 0);
    entries.forEach(e -> e.release());
    entries = c2.readEntries(2);
    assertEquals(entries.size(), 2);
    entries.forEach(e -> e.release());
    entries = c2.readEntries(2);
    assertEquals(entries.size(), 0);
    entries.forEach(e -> e.release());
}

Example 99 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class ManagedCursorTest method testSkipEntriesWithIndividualDeletedMessages.

@Test(timeOut = 20000)
void testSkipEntriesWithIndividualDeletedMessages() throws Exception {
    ManagedLedger ledger = factory.open("testSkipEntriesWithIndividualDeletedMessages", new ManagedLedgerConfig().setMaxEntriesPerLedger(5));
    ManagedCursor c1 = ledger.openCursor("c1");
    Position pos1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
    Position pos2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
    Position pos3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
    Position pos4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
    Position pos5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding));
    // delete individual messages
    c1.delete(pos2);
    c1.delete(pos4);
    c1.skipEntries(3, IndividualDeletedEntries.Exclude);
    assertEquals(c1.getNumberOfEntries(), 0);
    assertEquals(c1.getReadPosition(), pos5.getNext());
    assertEquals(c1.getMarkDeletedPosition(), pos5);
    pos1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
    pos2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
    pos3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
    pos4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
    pos5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding));
    c1.delete(pos2);
    c1.delete(pos4);
    c1.skipEntries(4, IndividualDeletedEntries.Include);
    assertEquals(c1.getNumberOfEntries(), 1);
    assertEquals(c1.getReadPosition(), pos5);
    assertEquals(c1.getMarkDeletedPosition(), pos4);
}

Example 100 with ManagedLedgerConfig

use of org.apache.bookkeeper.mledger.ManagedLedgerConfig in project incubator-pulsar by apache.

the class BrokerService method createPersistentTopic.

private void createPersistentTopic(final String topic, CompletableFuture<Topic> topicFuture) {
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    TopicName topicName = TopicName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(topicName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().submit(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        return;
    }
    getManagedLedgerConfig(topicName).thenAccept(managedLedgerConfig -> {
        // Once we have the configuration, we can proceed with the async open operation
        managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                try {
                    PersistentTopic persistentTopic = new PersistentTopic(topic, ledger, BrokerService.this);
                    CompletableFuture<Void> replicationFuture = persistentTopic.checkReplication();
                    replicationFuture.thenCompose(v -> {
                        // Also check dedup status
                        return persistentTopic.checkDeduplicationStatus();
                    }).thenRun(() -> {
                        log.info("Created topic {} - dedup is {}", topic, persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled");
                        long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                        pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                        addTopicToStatsMaps(topicName, persistentTopic);
                        topicFuture.complete(persistentTopic);
                    }).exceptionally((ex) -> {
                        log.warn("Replication or dedup check failed. Removing topic from topics list {}, {}", topic, ex);
                        persistentTopic.stopReplProducers().whenComplete((v, exception) -> {
                            topics.remove(topic, topicFuture);
                            topicFuture.completeExceptionally(ex);
                        });
                        return null;
                    });
                } catch (NamingException e) {
                    log.warn("Failed to create topic {}-{}", topic, e.getMessage());
                    pulsar.getExecutor().submit(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(e);
                }
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                log.warn("Failed to create topic {}", topic, exception);
                pulsar.getExecutor().submit(() -> topics.remove(topic, topicFuture));
                topicFuture.completeExceptionally(new PersistenceException(exception));
            }
        }, null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        // remove topic from topics-map in different thread to avoid possible deadlock if
        // createPersistentTopic-thread only tries to handle this future-result
        pulsar.getExecutor().submit(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(exception);
        return null;
    });
}