Examples with PositionImpl org.apache.bookkeeper.mledger.impl.PositionImpl used on opensource projects

Example 1 with PositionImpl

use of org.apache.bookkeeper.mledger.impl.PositionImpl in project pulsar by yahoo.

the class PersistentDispatcherMultipleConsumers method readEntriesFailed.

@Override
public synchronized void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
    ReadType readType = (ReadType) ctx;
    long waitTimeMillis = readFailureBackoff.next();
    if (!(exception instanceof TooManyRequestsException)) {
        log.error("[{}] Error reading entries at {} : {}, Read Type {} - Retrying to read in {} seconds", name, cursor.getReadPosition(), exception.getMessage(), readType, waitTimeMillis / 1000.0);
    } else {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Error reading entries at {} : {}, Read Type {} - Retrying to read in {} seconds", name, cursor.getReadPosition(), exception.getMessage(), readType, waitTimeMillis / 1000.0);
        }
    }
    if (shouldRewindBeforeReadingOrReplaying) {
        shouldRewindBeforeReadingOrReplaying = false;
        cursor.rewind();
    }
    if (readType == ReadType.Normal) {
        havePendingRead = false;
    } else {
        havePendingReplayRead = false;
        if (exception instanceof ManagedLedgerException.InvalidReplayPositionException) {
            PositionImpl markDeletePosition = (PositionImpl) cursor.getMarkDeletedPosition();
            messagesToReplay.removeIf(current -> current.compareTo(markDeletePosition) <= 0);
        }
    }
    readBatchSize = 1;
    topic.getBrokerService().executor().schedule(() -> {
        synchronized (PersistentDispatcherMultipleConsumers.this) {
            if (!havePendingRead) {
                log.info("[{}] Retrying read operation", name);
                readMoreEntries();
            } else {
                log.info("[{}] Skipping read retry: havePendingRead {}", name, havePendingRead, exception);
            }
        }
    }, waitTimeMillis, TimeUnit.MILLISECONDS);
}

Example 2 with PositionImpl

use of org.apache.bookkeeper.mledger.impl.PositionImpl in project pulsar by yahoo.

the class PersistentTopicTest method setupMLAsyncCallbackMocks.

void setupMLAsyncCallbackMocks() {
    ledgerMock = mock(ManagedLedger.class);
    cursorMock = mock(ManagedCursor.class);
    final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    doReturn("mockCursor").when(cursorMock).getName();
    // doNothing().when(cursorMock).asyncClose(new CloseCallback() {
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            // return closeFuture.get();
            return closeFuture.complete(null);
        }
    }).when(cursorMock).asyncClose(new CloseCallback() {

        @Override
        public void closeComplete(Object ctx) {
            log.info("[{}] Successfully closed cursor ledger", "mockCursor");
            closeFuture.complete(null);
        }

        @Override
        public void closeFailed(ManagedLedgerException exception, Object ctx) {
            // isFenced.set(false);
            log.error("Error closing cursor for subscription", exception);
            closeFuture.completeExceptionally(new BrokerServiceException.PersistenceException(exception));
        }
    }, null);
    // call openLedgerComplete with ledgerMock on ML factory asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((OpenLedgerCallback) invocationOnMock.getArguments()[2]).openLedgerComplete(ledgerMock, null);
            return null;
        }
    }).when(mlFactoryMock).asyncOpen(matches(".*success.*"), any(ManagedLedgerConfig.class), any(OpenLedgerCallback.class), anyObject());
    // call openLedgerFailed on ML factory asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((OpenLedgerCallback) invocationOnMock.getArguments()[2]).openLedgerFailed(new ManagedLedgerException("Managed ledger failure"), null);
            return null;
        }
    }).when(mlFactoryMock).asyncOpen(matches(".*fail.*"), any(ManagedLedgerConfig.class), any(OpenLedgerCallback.class), anyObject());
    // call addComplete on ledger asyncAddEntry
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((AddEntryCallback) invocationOnMock.getArguments()[1]).addComplete(new PositionImpl(1, 1), invocationOnMock.getArguments()[2]);
            return null;
        }
    }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), anyObject());
    // call openCursorComplete on cursor asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((OpenCursorCallback) invocationOnMock.getArguments()[1]).openCursorComplete(cursorMock, null);
            return null;
        }
    }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(OpenCursorCallback.class), anyObject());
    // call deleteLedgerComplete on ledger asyncDelete
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((DeleteLedgerCallback) invocationOnMock.getArguments()[0]).deleteLedgerComplete(null);
            return null;
        }
    }).when(ledgerMock).asyncDelete(any(DeleteLedgerCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((DeleteCursorCallback) invocationOnMock.getArguments()[1]).deleteCursorComplete(null);
            return null;
        }
    }).when(ledgerMock).asyncDeleteCursor(matches(".*success.*"), any(DeleteCursorCallback.class), anyObject());
}

Example 3 with PositionImpl

use of org.apache.bookkeeper.mledger.impl.PositionImpl in project pulsar by yahoo.

the class PersistentTopicE2ETest method testMessageReplay.

/**
     * Verify: 
     * 1. Broker should not replay already acknowledged messages 
     * 2. Dispatcher should not stuck while dispatching new messages due to previous-replay 
     * of invalid/already-acked messages
     * 
     * @throws Exception
     */
@Test
public void testMessageReplay() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int totalMessages = 10;
    int replayIndex = totalMessages / 2;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(1);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef.getDispatcher();
    Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToReplay");
    replayMap.setAccessible(true);
    TreeSet<PositionImpl> messagesToReplay = Sets.newTreeSet();
    assertNotNull(subRef);
    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    MessageIdImpl firstAckedMsg = null;
    // (2) Consume and ack messages except first message
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        if (i == 0) {
            firstAckedMsg = msgId;
        }
        if (i < replayIndex) {
            // (3) accumulate acked messages for replay
            messagesToReplay.add(new PositionImpl(msgId.getLedgerId(), msgId.getEntryId()));
        }
    }
    // (4) redelivery : should redeliver only unacked messages
    Thread.sleep(1000);
    replayMap.set(dispatcher, messagesToReplay);
    // (a) redelivery with all acked-message should clear messageReply bucket
    dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
    assertEquals(messagesToReplay.size(), 0);
    // (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
    messagesToReplay.add(new PositionImpl(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId()));
    replayMap.set(dispatcher, messagesToReplay);
    // send new message
    final String testMsg = "testMsg";
    producer.send(testMsg.getBytes());
    // consumer should be able to receive only new message and not the
    dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertNotNull(msg);
    assertEquals(msg.getData(), testMsg.getBytes());
    consumer.close();
    producer.close();
}

Example 4 with PositionImpl

use of org.apache.bookkeeper.mledger.impl.PositionImpl in project pulsar by yahoo.

the class PersistentTopics method peekNthMessage.

@GET
@Path("/{property}/{cluster}/{namespace}/{destination}/subscription/{subName}/position/{messagePosition}")
@ApiOperation(value = "Peek nth message on a topic subscription.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic, subscription or the message position does not exist") })
public Response peekNthMessage(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @PathParam("subName") String subName, @PathParam("messagePosition") int messagePosition, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    destination = decode(destination);
    DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace, destination, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnDestination(dn, authoritative);
    PersistentTopic topic = getTopicReference(dn);
    PersistentReplicator repl = null;
    PersistentSubscription sub = null;
    Entry entry = null;
    if (subName.startsWith(topic.replicatorPrefix)) {
        repl = getReplicatorReference(subName, topic);
    } else {
        sub = getSubscriptionReference(subName, topic);
    }
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            entry = repl.peekNthMessage(messagePosition).get();
        } else {
            entry = sub.peekNthMessage(messagePosition).get();
        }
        checkNotNull(entry);
        PositionImpl pos = (PositionImpl) entry.getPosition();
        ByteBuf metadataAndPayload = entry.getDataBuffer();
        // moves the readerIndex to the payload
        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
        ResponseBuilder responseBuilder = Response.ok();
        responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
        for (KeyValue keyValue : metadata.getPropertiesList()) {
            responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
        }
        if (metadata.hasPublishTime()) {
            responseBuilder.header("X-Pulsar-publish-time", DATE_FORMAT.format(Instant.ofEpochMilli(metadata.getPublishTime())));
        }
        // Decode if needed
        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
        ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
        // Copy into a heap buffer for output stream compatibility
        ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
        data.writeBytes(uncompressedPayload);
        uncompressedPayload.release();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream output) throws IOException, WebApplicationException {
                output.write(data.array(), data.arrayOffset(), data.readableBytes());
                data.release();
            }
        };
        return responseBuilder.entity(stream).build();
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Message not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, dn, subName, exception);
        throw new RestException(exception);
    } finally {
        if (entry != null) {
            entry.release();
        }
    }
}

Example 5 with PositionImpl

use of org.apache.bookkeeper.mledger.impl.PositionImpl in project pulsar by yahoo.

the class ServerCnxTest method setupMLAsyncCallbackMocks.

private void setupMLAsyncCallbackMocks() {
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    // call openLedgerComplete with ledgerMock on ML factory asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Thread.sleep(300);
            ((OpenLedgerCallback) invocationOnMock.getArguments()[2]).openLedgerComplete(ledgerMock, null);
            return null;
        }
    }).when(mlFactoryMock).asyncOpen(matches(".*success.*"), any(ManagedLedgerConfig.class), any(OpenLedgerCallback.class), anyObject());
    // call openLedgerFailed on ML factory asyncOpen
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Thread.sleep(300);
            new Thread(() -> {
                ((OpenLedgerCallback) invocationOnMock.getArguments()[2]).openLedgerFailed(new ManagedLedgerException("Managed ledger failure"), null);
            }).start();
            return null;
        }
    }).when(mlFactoryMock).asyncOpen(matches(".*fail.*"), any(ManagedLedgerConfig.class), any(OpenLedgerCallback.class), anyObject());
    // call addComplete on ledger asyncAddEntry
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((AddEntryCallback) invocationOnMock.getArguments()[1]).addComplete(new PositionImpl(-1, -1), invocationOnMock.getArguments()[2]);
            return null;
        }
    }).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Thread.sleep(300);
            ((OpenCursorCallback) invocationOnMock.getArguments()[1]).openCursorComplete(cursorMock, null);
            return null;
        }
    }).when(ledgerMock).asyncOpenCursor(matches(".*success.*"), any(OpenCursorCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Thread.sleep(300);
            ((OpenCursorCallback) invocationOnMock.getArguments()[1]).openCursorFailed(new ManagedLedgerException("Managed ledger failure"), null);
            return null;
        }
    }).when(ledgerMock).asyncOpenCursor(matches(".*fail.*"), any(OpenCursorCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((DeleteCursorCallback) invocationOnMock.getArguments()[1]).deleteCursorComplete(null);
            return null;
        }
    }).when(ledgerMock).asyncDeleteCursor(matches(".*success.*"), any(DeleteCursorCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((DeleteCursorCallback) invocationOnMock.getArguments()[1]).deleteCursorFailed(new ManagedLedgerException("Managed ledger failure"), null);
            return null;
        }
    }).when(ledgerMock).asyncDeleteCursor(matches(".*fail.*"), any(DeleteCursorCallback.class), anyObject());
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ((CloseCallback) invocationOnMock.getArguments()[0]).closeComplete(null);
            return null;
        }
    }).when(cursorMock).asyncClose(any(CloseCallback.class), anyObject());
    doReturn(successSubName).when(cursorMock).getName();
}