Examples with Position org.apache.bookkeeper.mledger.Position used on opensource projects

Example 86 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedCursorTest method testSkipEntriesWithIndividualDeletedMessages.

@Test(timeOut = 20000)
void testSkipEntriesWithIndividualDeletedMessages() throws Exception {
    ManagedLedger ledger = factory.open("testSkipEntriesWithIndividualDeletedMessages", new ManagedLedgerConfig().setMaxEntriesPerLedger(5));
    ManagedCursor c1 = ledger.openCursor("c1");
    Position pos1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
    Position pos2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
    Position pos3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
    Position pos4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
    Position pos5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding));
    // delete individual messages
    c1.delete(pos2);
    c1.delete(pos4);
    c1.skipEntries(3, IndividualDeletedEntries.Exclude);
    assertEquals(c1.getNumberOfEntries(), 0);
    assertEquals(c1.getReadPosition(), pos5.getNext());
    assertEquals(c1.getMarkDeletedPosition(), pos5);
    pos1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
    pos2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
    pos3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
    pos4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
    pos5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding));
    c1.delete(pos2);
    c1.delete(pos4);
    c1.skipEntries(4, IndividualDeletedEntries.Include);
    assertEquals(c1.getNumberOfEntries(), 1);
    assertEquals(c1.getReadPosition(), pos5);
    assertEquals(c1.getMarkDeletedPosition(), pos4);
}

Example 87 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedLedgerWriter method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-producer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    arguments.testTime = TimeUnit.SECONDS.toMillis(arguments.testTime);
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar managed-ledger perf writer with config: {}", w.writeValueAsString(arguments));
    byte[] payloadData = new byte[arguments.msgSize];
    ByteBuf payloadBuffer = PooledByteBufAllocator.DEFAULT.directBuffer(arguments.msgSize);
    payloadBuffer.writerIndex(arguments.msgSize);
    // Now processing command line arguments
    String managedLedgerPrefix = "test-" + DigestUtils.sha1Hex(UUID.randomUUID().toString()).substring(0, 5);
    ClientConfiguration bkConf = new ClientConfiguration();
    bkConf.setUseV2WireProtocol(true);
    bkConf.setLedgerManagerFactoryClass(HierarchicalLedgerManagerFactory.class);
    bkConf.setAddEntryTimeout(30);
    bkConf.setReadEntryTimeout(30);
    bkConf.setThrottleValue(0);
    bkConf.setNumChannelsPerBookie(arguments.maxConnections);
    bkConf.setZkServers(arguments.zookeeperServers);
    ManagedLedgerFactoryConfig mlFactoryConf = new ManagedLedgerFactoryConfig();
    mlFactoryConf.setMaxCacheSize(0);
    ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkConf, mlFactoryConf);
    ManagedLedgerConfig mlConf = new ManagedLedgerConfig();
    mlConf.setEnsembleSize(arguments.ensembleSize);
    mlConf.setWriteQuorumSize(arguments.writeQuorum);
    mlConf.setAckQuorumSize(arguments.ackQuorum);
    mlConf.setMinimumRolloverTime(10, TimeUnit.MINUTES);
    mlConf.setMetadataEnsembleSize(arguments.ensembleSize);
    mlConf.setMetadataWriteQuorumSize(arguments.writeQuorum);
    mlConf.setMetadataAckQuorumSize(arguments.ackQuorum);
    mlConf.setDigestType(arguments.digestType);
    mlConf.setMaxSizePerLedgerMb(2048);
    List<CompletableFuture<ManagedLedger>> futures = new ArrayList<>();
    for (int i = 0; i < arguments.numManagedLedgers; i++) {
        String name = String.format("%s-%03d", managedLedgerPrefix, i);
        CompletableFuture<ManagedLedger> future = new CompletableFuture<>();
        futures.add(future);
        factory.asyncOpen(name, mlConf, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                future.complete(ledger);
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                future.completeExceptionally(exception);
            }
        }, null);
    }
    List<ManagedLedger> managedLedgers = futures.stream().map(CompletableFuture::join).collect(Collectors.toList());
    log.info("Created {} managed ledgers", managedLedgers.size());
    Runtime.getRuntime().addShutdownHook(new Thread() {

        public void run() {
            printAggregatedStats();
        }
    });
    Collections.shuffle(managedLedgers);
    AtomicBoolean isDone = new AtomicBoolean();
    List<List<ManagedLedger>> managedLedgersPerThread = Lists.partition(managedLedgers, Math.max(1, managedLedgers.size() / arguments.numThreads));
    for (int i = 0; i < arguments.numThreads; i++) {
        List<ManagedLedger> managedLedgersForThisThread = managedLedgersPerThread.get(i);
        int nunManagedLedgersForThisThread = managedLedgersForThisThread.size();
        long numMessagesForThisThread = arguments.numMessages / arguments.numThreads;
        int maxOutstandingForThisThread = arguments.maxOutstanding;
        executor.submit(() -> {
            try {
                final double msgRate = arguments.msgRate / (double) arguments.numThreads;
                final RateLimiter rateLimiter = RateLimiter.create(msgRate);
                // Acquire 1 sec worth of messages to have a slower ramp-up
                rateLimiter.acquire((int) msgRate);
                final long startTime = System.currentTimeMillis();
                final Semaphore semaphore = new Semaphore(maxOutstandingForThisThread);
                final AddEntryCallback addEntryCallback = new AddEntryCallback() {

                    @Override
                    public void addComplete(Position position, Object ctx) {
                        long sendTime = (Long) (ctx);
                        messagesSent.increment();
                        bytesSent.add(payloadData.length);
                        long latencyMicros = NANOSECONDS.toMicros(System.nanoTime() - sendTime);
                        recorder.recordValue(latencyMicros);
                        cumulativeRecorder.recordValue(latencyMicros);
                        semaphore.release();
                    }

                    @Override
                    public void addFailed(ManagedLedgerException exception, Object ctx) {
                        log.warn("Write error on message", exception);
                        System.exit(-1);
                    }
                };
                // Send messages on all topics/producers
                long totalSent = 0;
                while (true) {
                    for (int j = 0; j < nunManagedLedgersForThisThread; j++) {
                        if (arguments.testTime > 0) {
                            if (System.currentTimeMillis() - startTime > arguments.testTime) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }
                        if (numMessagesForThisThread > 0) {
                            if (totalSent++ >= numMessagesForThisThread) {
                                log.info("------------------- DONE -----------------------");
                                printAggregatedStats();
                                isDone.set(true);
                                Thread.sleep(5000);
                                System.exit(0);
                            }
                        }
                        semaphore.acquire();
                        rateLimiter.acquire();
                        final long sendTime = System.nanoTime();
                        managedLedgersForThisThread.get(j).asyncAddEntry(payloadBuffer, addEntryCallback, sendTime);
                    }
                }
            } catch (Throwable t) {
                log.error("Got error", t);
            }
        });
    }
    // Print report stats
    long oldTime = System.nanoTime();
    Histogram reportHistogram = null;
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        if (isDone.get()) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesSent.sumThenReset() / elapsed;
        double throughput = bytesSent.sumThenReset() / elapsed / 1024 / 1024 * 8;
        reportHistogram = recorder.getIntervalHistogram(reportHistogram);
        log.info("Throughput produced: {}  msg/s --- {} Mbit/s --- Latency: mean: {} ms - med: {} - 95pct: {} - 99pct: {} - 99.9pct: {} - 99.99pct: {} - Max: {}", throughputFormat.format(rate), throughputFormat.format(throughput), dec.format(reportHistogram.getMean() / 1000.0), dec.format(reportHistogram.getValueAtPercentile(50) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(95) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.9) / 1000.0), dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0));
        reportHistogram.reset();
        oldTime = now;
    }
    factory.shutdown();
}

Example 88 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedLedgerImpl method asyncReadEntry.

void asyncReadEntry(PositionImpl position, ReadEntryCallback callback, Object ctx) {
    LedgerHandle currentLedger = this.currentLedger;
    if (log.isDebugEnabled()) {
        log.debug("[{}] Reading entry ledger {}: {}", name, position.getLedgerId(), position.getEntryId());
    }
    if (position.getLedgerId() == currentLedger.getId()) {
        LedgerHandle ledger = currentLedger;
        entryCache.asyncReadEntry(ledger, position, callback, ctx);
    } else {
        getLedgerHandle(position.getLedgerId()).thenAccept(ledger -> {
            entryCache.asyncReadEntry(ledger, position, callback, ctx);
        }).exceptionally(ex -> {
            log.error("[{}] Error opening ledger for reading at position {} - {}", name, position, ex.getMessage());
            callback.readEntryFailed(ManagedLedgerException.getManagedLedgerException(ex.getCause()), ctx);
            return null;
        });
    }
}

Example 89 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedLedgerImpl method getEstimatedBacklogSize.

@Override
public long getEstimatedBacklogSize() {
    PositionImpl pos = getMarkDeletePositionOfSlowestConsumer();
    while (true) {
        if (pos == null) {
            return 0;
        }
        long size = 0;
        final long slowestConsumerLedgerId = pos.getLedgerId();
        // Subtract size of ledgers that were already fully consumed but not trimmed yet
        synchronized (this) {
            size = getTotalSize();
            size -= ledgers.values().stream().filter(li -> li.getLedgerId() < slowestConsumerLedgerId).mapToLong(li -> li.getSize()).sum();
        }
        LedgerInfo ledgerInfo = null;
        synchronized (this) {
            ledgerInfo = ledgers.get(pos.getLedgerId());
        }
        if (ledgerInfo == null) {
            // ledger was removed
            if (pos.compareTo(getMarkDeletePositionOfSlowestConsumer()) == 0) {
                // position still has not moved
                return size;
            }
            // retry with new slowest consumer
            pos = getMarkDeletePositionOfSlowestConsumer();
            continue;
        }
        long numEntries = pos.getEntryId();
        if (ledgerInfo.getEntries() == 0) {
            size -= consumedLedgerSize(currentLedgerSize, currentLedgerEntries, numEntries);
            return size;
        } else {
            size -= consumedLedgerSize(ledgerInfo.getSize(), ledgerInfo.getEntries(), numEntries);
            return size;
        }
    }
}

Example 90 with Position

use of org.apache.bookkeeper.mledger.Position in project incubator-pulsar by apache.

the class ManagedLedgerImpl method asyncOpenCursor.

@Override
public synchronized void asyncOpenCursor(final String cursorName, final InitialPosition initialPosition, final OpenCursorCallback callback, final Object ctx) {
    try {
        checkManagedLedgerIsOpen();
        checkFenced();
    } catch (ManagedLedgerException e) {
        callback.openCursorFailed(e, ctx);
        return;
    }
    if (uninitializedCursors.containsKey(cursorName)) {
        uninitializedCursors.get(cursorName).thenAccept(cursor -> {
            callback.openCursorComplete(cursor, ctx);
        }).exceptionally(ex -> {
            callback.openCursorFailed((ManagedLedgerException) ex, ctx);
            return null;
        });
        return;
    }
    ManagedCursor cachedCursor = cursors.get(cursorName);
    if (cachedCursor != null) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Cursor was already created {}", name, cachedCursor);
        }
        callback.openCursorComplete(cachedCursor, ctx);
        return;
    }
    // Create a new one and persist it
    if (log.isDebugEnabled()) {
        log.debug("[{}] Creating new cursor: {}", name, cursorName);
    }
    final ManagedCursorImpl cursor = new ManagedCursorImpl(bookKeeper, config, this, cursorName);
    CompletableFuture<ManagedCursor> cursorFuture = new CompletableFuture<>();
    uninitializedCursors.put(cursorName, cursorFuture);
    cursor.initialize(getLastPosition(), new VoidCallback() {

        @Override
        public void operationComplete() {
            log.info("[{}] Opened new cursor: {}", name, cursor);
            cursor.setActive();
            // Update the ack position (ignoring entries that were written while the cursor was being created)
            cursor.initializeCursorPosition(initialPosition == InitialPosition.Latest ? getLastPositionAndCounter() : getFirstPositionAndCounter());
            synchronized (this) {
                cursors.add(cursor);
                uninitializedCursors.remove(cursorName).complete(cursor);
            }
            callback.openCursorComplete(cursor, ctx);
        }

        @Override
        public void operationFailed(ManagedLedgerException exception) {
            log.warn("[{}] Failed to open cursor: {}", name, cursor);
            synchronized (this) {
                uninitializedCursors.remove(cursorName).completeExceptionally(exception);
            }
            callback.openCursorFailed(exception, ctx);
        }
    });
}