Examples with PersistentReplicator com.yahoo.pulsar.broker.service.persistent.PersistentReplicator used on opensource projects

Example 1 with PersistentReplicator

use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.

the class ReplicatorTest method testReplicatorProducerClosing.

@Test(priority = 5)
public void testReplicatorProducerClosing() throws Exception {
    log.info("--- Starting ReplicatorTest::testDeleteReplicatorFailure ---");
    final String topicName = "persistent://pulsar/global/ns/repltopicbatch";
    final DestinationName dest = DestinationName.get(topicName);
    MessageProducer producer1 = new MessageProducer(url1, dest);
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topicName);
    final String replicatorClusterName = topic.getReplicators().keys().get(0);
    PersistentReplicator replicator = topic.getPersistentReplicator(replicatorClusterName);
    pulsar2.close();
    pulsar3.close();
    replicator.disconnect(false);
    Thread.sleep(100);
    Field field = PersistentReplicator.class.getDeclaredField("producer");
    field.setAccessible(true);
    ProducerImpl producer = (ProducerImpl) field.get(replicator);
    assertNull(producer);
}

Example 2 with PersistentReplicator

use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.

the class ReplicatorTest method testReplicatePeekAndSkip.

@Test
public void testReplicatePeekAndSkip() throws Exception {
    SortedSet<String> testDests = new TreeSet<String>();
    final DestinationName dest = DestinationName.get("persistent://pulsar/global/ns/peekAndSeekTopic");
    testDests.add(dest.toString());
    MessageProducer producer1 = new MessageProducer(url1, dest);
    MessageConsumer consumer1 = new MessageConsumer(url3, dest);
    // Produce from cluster1 and consume from the rest
    producer1.produce(2);
    producer1.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = topic.getReplicators().get(topic.getReplicators().keys().get(0));
    replicator.skipMessages(2);
    CompletableFuture<Entry> result = replicator.peekNthMessage(1);
    Entry entry = result.get(50, TimeUnit.MILLISECONDS);
    assertNull(entry);
    consumer1.close();
}

Example 3 with PersistentReplicator

use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.

the class ReplicatorTest method testResumptionAfterBacklogRelaxed.

/**
     * Issue #199
     *
     * It verifies that: if the remote cluster reaches backlog quota limit, replicator temporarily stops and once the
     * backlog drains it should resume replication.
     *
     * @throws Exception
     */
@Test(enabled = true, priority = -1)
public void testResumptionAfterBacklogRelaxed() throws Exception {
    List<RetentionPolicy> policies = Lists.newArrayList();
    policies.add(RetentionPolicy.producer_exception);
    policies.add(RetentionPolicy.producer_request_hold);
    for (RetentionPolicy policy : policies) {
        DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns1/%s", policy));
        // Producer on r1
        MessageProducer producer1 = new MessageProducer(url1, dest);
        // Consumer on r1
        MessageConsumer consumer1 = new MessageConsumer(url1, dest);
        // Consumer on r2
        MessageConsumer consumer2 = new MessageConsumer(url2, dest);
        // Replicator for r1 -> r2
        PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
        PersistentReplicator replicator = topic.getPersistentReplicator("r2");
        // Restrict backlog quota limit to 1
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1, policy));
        // Produce a message to r1, then it will be replicated to r2 and fulfill the backlog.
        producer1.produce(1);
        consumer1.receive(1);
        Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
        // Produce 9 messages to r1, then it will be pended because of the backlog limit excess
        producer1.produce(9);
        consumer1.receive(9);
        Thread.sleep(1000L);
        assertEquals(replicator.getStats().replicationBacklog, 9);
        // Relax backlog quota limit to 1G
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1024 * 1024 * 1024, policy));
        Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
        // The messages should be replicated to r2
        assertEquals(replicator.getStats().replicationBacklog, 0);
        consumer2.receive(1);
        consumer2.receive(9);
        if (!consumer2.drained()) {
            throw new Exception("consumer2 - unexpected message in queue");
        }
        producer1.close();
        consumer1.close();
        consumer2.close();
    }
}

Example 4 with PersistentReplicator

use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.

the class PersistentTopics method peekNthMessage.

@GET
@Path("/{property}/{cluster}/{namespace}/{destination}/subscription/{subName}/position/{messagePosition}")
@ApiOperation(value = "Peek nth message on a topic subscription.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic, subscription or the message position does not exist") })
public Response peekNthMessage(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @PathParam("subName") String subName, @PathParam("messagePosition") int messagePosition, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    destination = decode(destination);
    DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace, destination, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnDestination(dn, authoritative);
    PersistentTopic topic = getTopicReference(dn);
    PersistentReplicator repl = null;
    PersistentSubscription sub = null;
    Entry entry = null;
    if (subName.startsWith(topic.replicatorPrefix)) {
        repl = getReplicatorReference(subName, topic);
    } else {
        sub = getSubscriptionReference(subName, topic);
    }
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            entry = repl.peekNthMessage(messagePosition).get();
        } else {
            entry = sub.peekNthMessage(messagePosition).get();
        }
        checkNotNull(entry);
        PositionImpl pos = (PositionImpl) entry.getPosition();
        ByteBuf metadataAndPayload = entry.getDataBuffer();
        // moves the readerIndex to the payload
        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
        ResponseBuilder responseBuilder = Response.ok();
        responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
        for (KeyValue keyValue : metadata.getPropertiesList()) {
            responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
        }
        if (metadata.hasPublishTime()) {
            responseBuilder.header("X-Pulsar-publish-time", DATE_FORMAT.format(Instant.ofEpochMilli(metadata.getPublishTime())));
        }
        // Decode if needed
        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
        ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
        // Copy into a heap buffer for output stream compatibility
        ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
        data.writeBytes(uncompressedPayload);
        uncompressedPayload.release();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream output) throws IOException, WebApplicationException {
                output.write(data.array(), data.arrayOffset(), data.readableBytes());
                data.release();
            }
        };
        return responseBuilder.entity(stream).build();
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Message not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, dn, subName, exception);
        throw new RestException(exception);
    } finally {
        if (entry != null) {
            entry.release();
        }
    }
}

Example 5 with PersistentReplicator

use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.

the class PersistentTopicTest method testAtomicReplicationRemoval.

/**
     * {@link PersistentReplicator.removeReplicator} doesn't remove replicator in atomic way and does in multiple step:
     * 1. disconnect replicator producer
     * <p>
     * 2. close cursor
     * <p>
     * 3. remove from replicator-list.
     * <p>
     * 
     * If we try to startReplicationProducer before step-c finish then it should not avoid restarting repl-producer.
     * 
     * @throws Exception
     */
@Test
public void testAtomicReplicationRemoval() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns-abc/successTopic";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    String remoteReplicatorName = topic.replicatorPrefix + "." + remoteCluster;
    ConcurrentOpenHashMap<String, PersistentReplicator> replicatorMap = topic.getReplicators();
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = PulsarClient.create(brokerUrl.toString());
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = spy(new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService));
    replicatorMap.put(remoteReplicatorName, replicator);
    // step-1 remove replicator : it will disconnect the producer but it will wait for callback to be completed
    Method removeMethod = PersistentTopic.class.getDeclaredMethod("removeReplicator", String.class);
    removeMethod.setAccessible(true);
    removeMethod.invoke(topic, remoteReplicatorName);
    // step-2 now, policies doesn't have removed replication cluster so, it should not invoke "startProducer" of the
    // replicator
    when(pulsar.getConfigurationCache().policiesCache().get(AdminResource.path("policies", DestinationName.get(globalTopicName).getNamespace()))).thenReturn(Optional.of(new Policies()));
    // try to start replicator again
    topic.startReplProducers();
    // verify: replicator.startProducer is not invoked
    verify(replicator, Mockito.times(0)).startProducer();
    // step-3 : complete the callback to remove replicator from the list
    ArgumentCaptor<DeleteCursorCallback> captor = ArgumentCaptor.forClass(DeleteCursorCallback.class);
    Mockito.verify(ledgerMock).asyncDeleteCursor(anyObject(), captor.capture(), anyObject());
    DeleteCursorCallback callback = captor.getValue();
    callback.deleteCursorComplete(null);
}