Examples with PersistentReplicator org.apache.pulsar.broker.service.persistent.PersistentReplicator used on opensource projects

Example 1 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalSkipMessages.

protected void internalSkipMessages(String subName, int numMessages, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Skip messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnTopic(authoritative);
    PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            String remoteCluster = PersistentReplicator.getRemoteCluster(subName);
            PersistentReplicator repl = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster);
            checkNotNull(repl);
            repl.skipMessages(numMessages).get();
        } else {
            PersistentSubscription sub = topic.getSubscription(subName);
            checkNotNull(sub);
            sub.skipMessages(numMessages).get();
        }
        log.info("[{}] Skipped {} messages on {} {}", clientAppId(), numMessages, topicName, subName);
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Subscription not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to skip {} messages {} {}", clientAppId(), numMessages, topicName, subName, exception);
        throw new RestException(exception);
    }
}

Example 2 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalExpireMessages.

protected void internalExpireMessages(String subName, int expireTimeInSeconds, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        // expire messages for each partition topic
        try {
            for (int i = 0; i < partitionMetadata.partitions; i++) {
                pulsar().getAdminClient().persistentTopics().expireMessages(topicName.getPartition(i).toString(), subName, expireTimeInSeconds);
            }
        } catch (Exception e) {
            throw new RestException(e);
        }
    } else {
        // validate ownership and redirect if current broker is not owner
        validateAdminOperationOnTopic(authoritative);
        if (!(getTopicReference(topicName) instanceof PersistentTopic)) {
            log.error("[{}] Not supported operation of non-persistent topic {} {}", clientAppId(), topicName, subName);
            throw new RestException(Status.METHOD_NOT_ALLOWED, "Expire messages on a non-persistent topic is not allowed");
        }
        PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
        try {
            if (subName.startsWith(topic.replicatorPrefix)) {
                String remoteCluster = PersistentReplicator.getRemoteCluster(subName);
                PersistentReplicator repl = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster);
                checkNotNull(repl);
                repl.expireMessages(expireTimeInSeconds);
            } else {
                PersistentSubscription sub = topic.getSubscription(subName);
                checkNotNull(sub);
                sub.expireMessages(expireTimeInSeconds);
            }
            log.info("[{}] Message expire started up to {} on {} {}", clientAppId(), expireTimeInSeconds, topicName, subName);
        } catch (NullPointerException npe) {
            throw new RestException(Status.NOT_FOUND, "Subscription not found");
        } catch (Exception exception) {
            log.error("[{}] Failed to expire messages up to {} on {} with subscription {} {}", clientAppId(), expireTimeInSeconds, topicName, subName, exception);
            throw new RestException(exception);
        }
    }
}

Example 3 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalPeekNthMessage.

protected Response internalPeekNthMessage(String subName, int messagePosition, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnTopic(authoritative);
    if (!(getTopicReference(topicName) instanceof PersistentTopic)) {
        log.error("[{}] Not supported operation of non-persistent topic {} {}", clientAppId(), topicName, subName);
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Skip messages on a non-persistent topic is not allowed");
    }
    PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
    PersistentReplicator repl = null;
    PersistentSubscription sub = null;
    Entry entry = null;
    if (subName.startsWith(topic.replicatorPrefix)) {
        repl = getReplicatorReference(subName, topic);
    } else {
        sub = (PersistentSubscription) getSubscriptionReference(subName, topic);
    }
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            entry = repl.peekNthMessage(messagePosition).get();
        } else {
            entry = sub.peekNthMessage(messagePosition).get();
        }
        checkNotNull(entry);
        PositionImpl pos = (PositionImpl) entry.getPosition();
        ByteBuf metadataAndPayload = entry.getDataBuffer();
        // moves the readerIndex to the payload
        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
        ResponseBuilder responseBuilder = Response.ok();
        responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
        for (KeyValue keyValue : metadata.getPropertiesList()) {
            responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
        }
        if (metadata.hasPublishTime()) {
            responseBuilder.header("X-Pulsar-publish-time", DateFormatter.format(metadata.getPublishTime()));
        }
        if (metadata.hasEventTime()) {
            responseBuilder.header("X-Pulsar-event-time", DateFormatter.format(metadata.getEventTime()));
        }
        if (metadata.hasNumMessagesInBatch()) {
            responseBuilder.header("X-Pulsar-num-batch-message", metadata.getNumMessagesInBatch());
        }
        // Decode if needed
        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
        ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
        // Copy into a heap buffer for output stream compatibility
        ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
        data.writeBytes(uncompressedPayload);
        uncompressedPayload.release();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream output) throws IOException, WebApplicationException {
                output.write(data.array(), data.arrayOffset(), data.readableBytes());
                data.release();
            }
        };
        return responseBuilder.entity(stream).build();
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Message not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, topicName, subName, exception);
        throw new RestException(exception);
    } finally {
        if (entry != null) {
            entry.release();
        }
    }
}

Example 4 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class PersistentTopicTest method testClosingReplicationProducerTwice.

@SuppressWarnings("unchecked")
@Test
public void testClosingReplicationProducerTwice() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns/testClosingReplicationProducerTwice";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = spy(PulsarClient.builder().serviceUrl(brokerUrl.toString()).build());
    PulsarClientImpl clientImpl = (PulsarClientImpl) client;
    doReturn(new CompletableFuture<Producer>()).when(clientImpl).createProducerAsync(any(ProducerConfigurationData.class), any(Schema.class));
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService);
    // PersistentReplicator constructor calls startProducer()
    verify(clientImpl).createProducerAsync(any(ProducerConfigurationData.class), any(Schema.class));
    replicator.disconnect(false);
    replicator.disconnect(false);
    replicator.startProducer();
    verify(clientImpl, Mockito.times(2)).createProducerAsync(any(ProducerConfigurationData.class), any(Schema.class));
}

Example 5 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class ReplicatorTest method testCloseReplicatorStartProducer.

/**
 * It verifies that PersistentReplicator considers CursorAlreadyClosedException as non-retriable-read exception and
 * it should closed the producer as cursor is already closed because replicator is already deleted.
 *
 * @throws Exception
 */
@Test(timeOut = 15000)
public void testCloseReplicatorStartProducer() throws Exception {
    TopicName dest = TopicName.get("persistent://pulsar/global/ns1/closeCursor");
    // Producer on r1
    MessageProducer producer1 = new MessageProducer(url1, dest);
    // Consumer on r1
    MessageConsumer consumer1 = new MessageConsumer(url1, dest);
    // Consumer on r2
    MessageConsumer consumer2 = new MessageConsumer(url2, dest);
    // Replicator for r1 -> r2
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = (PersistentReplicator) topic.getPersistentReplicator("r2");
    // close the cursor
    Field cursorField = PersistentReplicator.class.getDeclaredField("cursor");
    cursorField.setAccessible(true);
    ManagedCursor cursor = (ManagedCursor) cursorField.get(replicator);
    cursor.close();
    // try to read entries
    CountDownLatch latch = new CountDownLatch(1);
    producer1.produce(10);
    cursor.asyncReadEntriesOrWait(10, new ReadEntriesCallback() {

        @Override
        public void readEntriesComplete(List<Entry> entries, Object ctx) {
            latch.countDown();
            fail("it should have been failed");
        }

        @Override
        public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
            latch.countDown();
            assertTrue(exception instanceof CursorAlreadyClosedException);
        }
    }, null);
    // replicator-readException: cursorAlreadyClosed
    replicator.readEntriesFailed(new CursorAlreadyClosedException("Cursor already closed exception"), null);
    // wait replicator producer to be closed
    Thread.sleep(1000);
    // Replicator producer must be closed
    Field producerField = AbstractReplicator.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    @SuppressWarnings("unchecked") ProducerImpl<byte[]> replicatorProducer = (ProducerImpl<byte[]>) producerField.get(replicator);
    assertEquals(replicatorProducer, null);
    producer1.close();
    consumer1.close();
    consumer2.close();
}