Examples with PersistentTopic com.yahoo.pulsar.broker.service.persistent.PersistentTopic used on opensource projects

Example 36 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BrokerService method createPersistentTopic.

private CompletableFuture<Topic> createPersistentTopic(final String topic) throws RuntimeException {
    checkTopicNsOwnership(topic);
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    DestinationName destinationName = DestinationName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(destinationName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        throw new RuntimeException(new ServiceUnitNotReadyException(msg));
    }
    final CompletableFuture<Topic> topicFuture = new CompletableFuture<>();
    getManagedLedgerConfig(destinationName).thenAccept(config -> {
        managedLedgerFactory.asyncOpen(destinationName.getPersistenceNamingEncoding(), config, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                PersistentTopic persistentTopic = new PersistentTopic(topic, ledger, BrokerService.this);
                CompletableFuture<Void> replicationFuture = persistentTopic.checkReplication();
                replicationFuture.thenRun(() -> {
                    log.info("Created topic {}", topic);
                    long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                    pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                    addTopicToStatsMaps(destinationName, persistentTopic);
                    topicFuture.complete(persistentTopic);
                });
                replicationFuture.exceptionally((ex) -> {
                    log.warn("Replication check failed. Removing topic from topics list {}, {}", topic, ex);
                    persistentTopic.stopReplProducers().whenComplete((v, exception) -> {
                        topics.remove(topic, topicFuture);
                        topicFuture.completeExceptionally(ex);
                    });
                    return null;
                });
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                log.warn("Failed to create topic {}", topic, exception);
                topics.remove(topic, topicFuture);
                topicFuture.completeExceptionally(new PersistenceException(exception));
            }
        }, null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        topics.remove(topic, topicFuture);
        topicFuture.completeExceptionally(exception);
        return null;
    });
    return topicFuture;
}

Example 37 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class BrokerService method getTopicStats.

public Map<String, PersistentTopicStats> getTopicStats() {
    HashMap<String, PersistentTopicStats> stats = new HashMap<>();
    topics.forEach((name, topicFuture) -> {
        PersistentTopic currentTopic = (PersistentTopic) topicFuture.getNow(null);
        if (currentTopic != null) {
            stats.put(name, currentTopic.getStats());
        }
    });
    return stats;
}

Example 38 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicTest method testAtomicReplicationRemoval.

/**
     * {@link PersistentReplicator.removeReplicator} doesn't remove replicator in atomic way and does in multiple step:
     * 1. disconnect replicator producer
     * <p>
     * 2. close cursor
     * <p>
     * 3. remove from replicator-list.
     * <p>
     * 
     * If we try to startReplicationProducer before step-c finish then it should not avoid restarting repl-producer.
     * 
     * @throws Exception
     */
@Test
public void testAtomicReplicationRemoval() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns-abc/successTopic";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    String remoteReplicatorName = topic.replicatorPrefix + "." + remoteCluster;
    ConcurrentOpenHashMap<String, PersistentReplicator> replicatorMap = topic.getReplicators();
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = PulsarClient.create(brokerUrl.toString());
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = spy(new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService));
    replicatorMap.put(remoteReplicatorName, replicator);
    // step-1 remove replicator : it will disconnect the producer but it will wait for callback to be completed
    Method removeMethod = PersistentTopic.class.getDeclaredMethod("removeReplicator", String.class);
    removeMethod.setAccessible(true);
    removeMethod.invoke(topic, remoteReplicatorName);
    // step-2 now, policies doesn't have removed replication cluster so, it should not invoke "startProducer" of the
    // replicator
    when(pulsar.getConfigurationCache().policiesCache().get(AdminResource.path("policies", DestinationName.get(globalTopicName).getNamespace()))).thenReturn(Optional.of(new Policies()));
    // try to start replicator again
    topic.startReplProducers();
    // verify: replicator.startProducer is not invoked
    verify(replicator, Mockito.times(0)).startProducer();
    // step-3 : complete the callback to remove replicator from the list
    ArgumentCaptor<DeleteCursorCallback> captor = ArgumentCaptor.forClass(DeleteCursorCallback.class);
    Mockito.verify(ledgerMock).asyncDeleteCursor(anyObject(), captor.capture(), anyObject());
    DeleteCursorCallback callback = captor.getValue();
    callback.deleteCursorComplete(null);
}

Example 39 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicTest method testClosingReplicationProducerTwice.

@Test
public void testClosingReplicationProducerTwice() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns/testClosingReplicationProducerTwice";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    String remoteReplicatorName = topic.replicatorPrefix + "." + localCluster;
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = spy(PulsarClient.create(brokerUrl.toString()));
    PulsarClientImpl clientImpl = (PulsarClientImpl) client;
    Field conf = PersistentReplicator.class.getDeclaredField("producerConfiguration");
    conf.setAccessible(true);
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService);
    doReturn(new CompletableFuture<Producer>()).when(clientImpl).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
    replicator.startProducer();
    verify(clientImpl).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
    replicator.disconnect(false);
    replicator.disconnect(false);
    replicator.startProducer();
    verify(clientImpl, Mockito.times(2)).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
}

Example 40 with PersistentTopic

use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.

the class PersistentTopicTest method testDeleteTopicRaceConditions.

@Test
public void testDeleteTopicRaceConditions() throws Exception {
    PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    // override ledger deletion callback to slow down deletion
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            Thread.sleep(1000);
            ((DeleteLedgerCallback) invocationOnMock.getArguments()[0]).deleteLedgerComplete(null);
            return null;
        }
    }).when(ledgerMock).asyncDelete(any(DeleteLedgerCallback.class), anyObject());
    ExecutorService executor = Executors.newCachedThreadPool();
    executor.submit(() -> {
        topic.delete();
        return null;
    }).get();
    try {
        String role = "appid1";
        Thread.sleep(10);
        /* delay to ensure that the delete gets executed first */
        Producer producer = new Producer(topic, serverCnx, 1, /* producer id */
        "prod-name", role);
        topic.addProducer(producer);
        fail("Should have failed");
    } catch (BrokerServiceException e) {
        assertTrue(e instanceof BrokerServiceException.TopicFencedException);
    }
    CommandSubscribe cmd = CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(SubType.Exclusive).build();
    Future<Consumer> f = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    try {
        f.get();
        fail("should have failed");
    } catch (ExecutionException ee) {
        assertTrue(ee.getCause() instanceof BrokerServiceException.TopicFencedException);
    // Expected
    }
}