Examples with BookKeeper org.apache.bookkeeper.client.BookKeeper used on opensource projects

Example 41 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class CompactedTopicTest method testEntryLookup.

@Test
public void testEntryLookup() throws Exception {
    BookKeeper bk = pulsar.getBookKeeperClientFactory().create(this.conf, null);
    Triple<Long, List<Pair<MessageIdData, Long>>, List<Pair<MessageIdData, Long>>> compactedLedgerData = buildCompactedLedger(bk, 500);
    List<Pair<MessageIdData, Long>> positions = compactedLedgerData.getMiddle();
    List<Pair<MessageIdData, Long>> idsInGaps = compactedLedgerData.getRight();
    LedgerHandle lh = bk.openLedger(compactedLedgerData.getLeft(), Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    long lastEntryId = lh.getLastAddConfirmed();
    AsyncLoadingCache<Long, MessageIdData> cache = CompactedTopicImpl.createCache(lh, 50);
    MessageIdData firstPositionId = positions.get(0).getLeft();
    Pair<MessageIdData, Long> lastPosition = positions.get(positions.size() - 1);
    // check ids before and after ids in compacted ledger
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(0, 0), lastEntryId, cache).get(), Long.valueOf(0));
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(Long.MAX_VALUE, 0), lastEntryId, cache).get(), Long.valueOf(CompactedTopicImpl.NEWER_THAN_COMPACTED));
    // entry 0 is never in compacted ledger due to how we generate dummy
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(firstPositionId.getLedgerId(), 0), lastEntryId, cache).get(), Long.valueOf(0));
    // check next id after last id in compacted ledger
    Assert.assertEquals(CompactedTopicImpl.findStartPoint(new PositionImpl(lastPosition.getLeft().getLedgerId(), lastPosition.getLeft().getEntryId() + 1), lastEntryId, cache).get(), Long.valueOf(CompactedTopicImpl.NEWER_THAN_COMPACTED));
    // shuffle to make cache work hard
    Collections.shuffle(positions, r);
    Collections.shuffle(idsInGaps, r);
    // Check ids we know are in compacted ledger
    for (Pair<MessageIdData, Long> p : positions) {
        PositionImpl pos = new PositionImpl(p.getLeft().getLedgerId(), p.getLeft().getEntryId());
        Long got = CompactedTopicImpl.findStartPoint(pos, lastEntryId, cache).get();
        Assert.assertEquals(got, Long.valueOf(p.getRight()));
    }
    // Check ids we know are in the gaps of the compacted ledger
    for (Pair<MessageIdData, Long> gap : idsInGaps) {
        PositionImpl pos = new PositionImpl(gap.getLeft().getLedgerId(), gap.getLeft().getEntryId());
        Assert.assertEquals(CompactedTopicImpl.findStartPoint(pos, lastEntryId, cache).get(), Long.valueOf(gap.getRight()));
    }
}

Example 42 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class CompactorTest method testCompactEmptyTopic.

@Test(expectedExceptions = ExecutionException.class)
public void testCompactEmptyTopic() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";
    // trigger creation of topic on server side
    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").subscribe().close();
    BookKeeper bk = pulsar.getBookKeeperClientFactory().create(this.conf, null);
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();
}

Example 43 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class ManagedLedgerImpl method asyncOpenCursor.

@Override
public synchronized void asyncOpenCursor(final String cursorName, final InitialPosition initialPosition, final OpenCursorCallback callback, final Object ctx) {
    try {
        checkManagedLedgerIsOpen();
        checkFenced();
    } catch (ManagedLedgerException e) {
        callback.openCursorFailed(e, ctx);
        return;
    }
    if (uninitializedCursors.containsKey(cursorName)) {
        uninitializedCursors.get(cursorName).thenAccept(cursor -> {
            callback.openCursorComplete(cursor, ctx);
        }).exceptionally(ex -> {
            callback.openCursorFailed((ManagedLedgerException) ex, ctx);
            return null;
        });
        return;
    }
    ManagedCursor cachedCursor = cursors.get(cursorName);
    if (cachedCursor != null) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Cursor was already created {}", name, cachedCursor);
        }
        callback.openCursorComplete(cachedCursor, ctx);
        return;
    }
    // Create a new one and persist it
    if (log.isDebugEnabled()) {
        log.debug("[{}] Creating new cursor: {}", name, cursorName);
    }
    final ManagedCursorImpl cursor = new ManagedCursorImpl(bookKeeper, config, this, cursorName);
    CompletableFuture<ManagedCursor> cursorFuture = new CompletableFuture<>();
    uninitializedCursors.put(cursorName, cursorFuture);
    cursor.initialize(getLastPosition(), new VoidCallback() {

        @Override
        public void operationComplete() {
            log.info("[{}] Opened new cursor: {}", name, cursor);
            cursor.setActive();
            // Update the ack position (ignoring entries that were written while the cursor was being created)
            cursor.initializeCursorPosition(initialPosition == InitialPosition.Latest ? getLastPositionAndCounter() : getFirstPositionAndCounter());
            synchronized (this) {
                cursors.add(cursor);
                uninitializedCursors.remove(cursorName).complete(cursor);
            }
            callback.openCursorComplete(cursor, ctx);
        }

        @Override
        public void operationFailed(ManagedLedgerException exception) {
            log.warn("[{}] Failed to open cursor: {}", name, cursor);
            synchronized (this) {
                uninitializedCursors.remove(cursorName).completeExceptionally(exception);
            }
            callback.openCursorFailed(exception, ctx);
        }
    });
}

Example 44 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class BrokerBkEnsemblesTests method testSkipCorruptDataLedger.

/**
 * It verifies broker-configuration using which broker can skip non-recoverable data-ledgers.
 *
 * <pre>
 * 1. publish messages in 5 data-ledgers each with 20 entries under managed-ledger
 * 2. delete first 4 data-ledgers
 * 3. consumer will fail to consume any message as first data-ledger is non-recoverable
 * 4. enable dynamic config to skip non-recoverable data-ledgers
 * 5. consumer will be able to consume 20 messages from last non-deleted ledger
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 6000)
public void testSkipCorruptDataLedger() throws Exception {
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final String ns1 = "prop/usc/crash-broker";
    final int totalMessages = 100;
    final int totalDataLedgers = 5;
    final int entriesPerLedger = totalMessages / totalDataLedgers;
    admin.namespaces().createNamespace(ns1);
    final String topic1 = "persistent://" + ns1 + "/my-topic";
    // Create subscription
    Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").receiverQueueSize(5).subscribe();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopic(topic1).get();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) topic.getManagedLedger();
    ManagedCursorImpl cursor = (ManagedCursorImpl) ml.getCursors().iterator().next();
    Field configField = ManagedCursorImpl.class.getDeclaredField("config");
    configField.setAccessible(true);
    // Create multiple data-ledger
    ManagedLedgerConfig config = (ManagedLedgerConfig) configField.get(cursor);
    config.setMaxEntriesPerLedger(entriesPerLedger);
    config.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS);
    // bookkeeper client
    Field bookKeeperField = ManagedLedgerImpl.class.getDeclaredField("bookKeeper");
    bookKeeperField.setAccessible(true);
    // Create multiple data-ledger
    BookKeeper bookKeeper = (BookKeeper) bookKeeperField.get(ml);
    // (1) publish messages in 5 data-ledgers each with 20 entries under managed-ledger
    Producer<byte[]> producer = client.newProducer().topic(topic1).create();
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // validate: consumer is able to consume msg and close consumer after reading 1 entry
    Assert.assertNotNull(consumer.receive(1, TimeUnit.SECONDS));
    consumer.close();
    NavigableMap<Long, LedgerInfo> ledgerInfo = ml.getLedgersInfo();
    Assert.assertEquals(ledgerInfo.size(), totalDataLedgers);
    Entry<Long, LedgerInfo> lastLedger = ledgerInfo.lastEntry();
    // (2) delete first 4 data-ledgers
    ledgerInfo.entrySet().forEach(entry -> {
        if (!entry.equals(lastLedger)) {
            try {
                bookKeeper.deleteLedger(entry.getKey());
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    });
    // clean managed-ledger and recreate topic to clean any data from the cache
    producer.close();
    pulsar.getBrokerService().removeTopicFromCache(topic1);
    ManagedLedgerFactoryImpl factory = (ManagedLedgerFactoryImpl) pulsar.getManagedLedgerFactory();
    Field field = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
    field.setAccessible(true);
    @SuppressWarnings("unchecked") ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) field.get(factory);
    ledgers.clear();
    // (3) consumer will fail to consume any message as first data-ledger is non-recoverable
    Message<byte[]> msg = null;
    // start consuming message
    consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    msg = consumer.receive(1, TimeUnit.SECONDS);
    Assert.assertNull(msg);
    consumer.close();
    // (4) enable dynamic config to skip non-recoverable data-ledgers
    admin.brokers().updateDynamicConfiguration("autoSkipNonRecoverableData", "true");
    retryStrategically((test) -> config.isAutoSkipNonRecoverableData(), 5, 100);
    // (5) consumer will be able to consume 20 messages from last non-deleted ledger
    consumer = client.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    for (int i = 0; i < entriesPerLedger; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        System.out.println(i);
        consumer.acknowledge(msg);
    }
    producer.close();
    consumer.close();
    client.close();
}

Example 45 with BookKeeper

use of org.apache.bookkeeper.client.BookKeeper in project incubator-pulsar by apache.

the class CompactorTool method main.

public static void main(String[] args) throws Exception {
    Arguments arguments = new Arguments();
    JCommander jcommander = new JCommander(arguments);
    jcommander.setProgramName("PulsarTopicCompactor");
    // parse args by JCommander
    jcommander.parse(args);
    if (arguments.help) {
        jcommander.usage();
        System.exit(-1);
    }
    // init broker config
    ServiceConfiguration brokerConfig;
    if (isBlank(arguments.brokerConfigFile)) {
        jcommander.usage();
        throw new IllegalArgumentException("Need to specify a configuration file for broker");
    } else {
        brokerConfig = PulsarConfigurationLoader.create(arguments.brokerConfigFile, ServiceConfiguration.class);
    }
    String pulsarServiceUrl = PulsarService.brokerUrl(brokerConfig);
    ClientConfiguration clientConfig = new ClientConfiguration();
    if (isNotBlank(brokerConfig.getBrokerClientAuthenticationPlugin())) {
        clientConfig.setAuthentication(brokerConfig.getBrokerClientAuthenticationPlugin(), brokerConfig.getBrokerClientAuthenticationParameters());
    }
    clientConfig.setUseTls(brokerConfig.isTlsEnabled());
    clientConfig.setTlsAllowInsecureConnection(brokerConfig.isTlsAllowInsecureConnection());
    clientConfig.setTlsTrustCertsFilePath(brokerConfig.getTlsCertificateFilePath());
    ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder().setNameFormat("compaction-%d").setDaemon(true).build());
    OrderedScheduler executor = OrderedScheduler.newSchedulerBuilder().build();
    ZooKeeperClientFactory zkClientFactory = new ZookeeperBkClientFactoryImpl(executor);
    ZooKeeper zk = zkClientFactory.create(brokerConfig.getZookeeperServers(), ZooKeeperClientFactory.SessionType.ReadWrite, (int) brokerConfig.getZooKeeperSessionTimeoutMillis()).get();
    BookKeeperClientFactory bkClientFactory = new BookKeeperClientFactoryImpl();
    BookKeeper bk = bkClientFactory.create(brokerConfig, zk);
    try (PulsarClient pulsar = PulsarClient.create(pulsarServiceUrl, clientConfig)) {
        Compactor compactor = new TwoPhaseCompactor(brokerConfig, pulsar, bk, scheduler);
        long ledgerId = compactor.compact(arguments.topic).get();
        log.info("Compaction of topic {} complete. Compacted to ledger {}", arguments.topic, ledgerId);
    } finally {
        bk.close();
        bkClientFactory.close();
        zk.close();
        scheduler.shutdownNow();
        executor.shutdown();
    }
}