Search in sources :

Example 11 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class MessageIdSerialization method testProtobufSerialization2.

@Test
void testProtobufSerialization2() throws Exception {
    MessageId id = new MessageIdImpl(1, 2, -1);
    byte[] serializedId = id.toByteArray();
    assertEquals(MessageId.fromByteArray(serializedId), id);
}
Also used : MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) MessageId(org.apache.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Example 12 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class MessageIdSerialization method testProtobufSerialization1.

@Test
void testProtobufSerialization1() throws Exception {
    MessageId id = new MessageIdImpl(1, 2, 3);
    byte[] serializedId = id.toByteArray();
    assertEquals(MessageId.fromByteArray(serializedId), id);
}
Also used : MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) MessageId(org.apache.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Example 13 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class PersistentTopic method terminate.

public CompletableFuture<MessageId> terminate() {
    CompletableFuture<MessageId> future = new CompletableFuture<>();
    ledger.asyncTerminate(new TerminateCallback() {

        @Override
        public void terminateComplete(Position lastCommittedPosition, Object ctx) {
            producers.forEach(Producer::disconnect);
            subscriptions.forEach((name, sub) -> sub.topicTerminated());
            PositionImpl lastPosition = (PositionImpl) lastCommittedPosition;
            MessageId messageId = new MessageIdImpl(lastPosition.getLedgerId(), lastPosition.getEntryId(), -1);
            log.info("[{}] Topic terminated at {}", getName(), messageId);
            future.complete(messageId);
        }

        @Override
        public void terminateFailed(ManagedLedgerException exception, Object ctx) {
            future.completeExceptionally(exception);
        }
    }, null);
    return future;
}
Also used : SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) Topic(org.apache.pulsar.broker.service.Topic) LedgerInfo(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats.LedgerInfo) PersistentTopicStats(org.apache.pulsar.common.policies.data.PersistentTopicStats) AdminResource(org.apache.pulsar.broker.admin.AdminResource) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) SchemaVersion(org.apache.pulsar.common.schema.SchemaVersion) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) LoggerFactory(org.slf4j.LoggerFactory) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) ProducerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ProducerBusyException) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) Map(java.util.Map) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) StatsOutputStream(org.apache.pulsar.utils.StatsOutputStream) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) Set(java.util.Set) Position(org.apache.bookkeeper.mledger.Position) BrokerService(org.apache.pulsar.broker.service.BrokerService) ReplicatorStats(org.apache.pulsar.common.policies.data.ReplicatorStats) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) NamespaceStats(org.apache.pulsar.broker.stats.NamespaceStats) Sets(com.google.common.collect.Sets) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) ServerCnx(org.apache.pulsar.broker.service.ServerCnx) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) ConcurrentOpenHashSet(org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) SchemaData(org.apache.pulsar.common.schema.SchemaData) Optional(java.util.Optional) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) PublisherStats(org.apache.pulsar.common.policies.data.PublisherStats) Subscription(org.apache.pulsar.broker.service.Subscription) TopicName(org.apache.pulsar.common.naming.TopicName) CursorStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats.CursorStats) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) Consumer(org.apache.pulsar.broker.service.Consumer) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) DateFormatter(org.apache.pulsar.common.util.DateFormatter) Lists(com.google.common.collect.Lists) ByteBuf(io.netty.buffer.ByteBuf) CompactedTopicImpl(org.apache.pulsar.compaction.CompactedTopicImpl) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) Logger(org.slf4j.Logger) SubType(org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.SubType) KeeperException(org.apache.zookeeper.KeeperException) CompactedTopic(org.apache.pulsar.compaction.CompactedTopic) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) MoreObjects(com.google.common.base.MoreObjects) AtomicLongFieldUpdater(java.util.concurrent.atomic.AtomicLongFieldUpdater) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Producer(org.apache.pulsar.broker.service.Producer) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) Compactor(org.apache.pulsar.compaction.Compactor) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) POLICIES(org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES) Codec(org.apache.pulsar.common.util.Codec) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) Replicator(org.apache.pulsar.broker.service.Replicator) ConsumerStats(org.apache.pulsar.common.policies.data.ConsumerStats) InitialPosition(org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition) Collections(java.util.Collections) CompletableFuture(java.util.concurrent.CompletableFuture) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) Position(org.apache.bookkeeper.mledger.Position) InitialPosition(org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageId(org.apache.pulsar.client.api.MessageId)

Example 14 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class AdminApiTest2 method testResetCursorOnPosition.

/**
 * Verifies reset-cursor at specific position using admin-api.
 *
 * <pre>
 * 1. Publish 50 messages
 * 2. Consume 20 messages
 * 3. reset cursor position on 10th message
 * 4. consume 40 messages from reset position
 * </pre>
 *
 * @param namespaceName
 * @throws Exception
 */
@Test(dataProvider = "namespaceNames", timeOut = 10000)
public void testResetCursorOnPosition(String namespaceName) throws Exception {
    final String topicName = "persistent://prop-xyz/use/" + namespaceName + "/resetPosition";
    final int totalProducedMessages = 50;
    // set retention
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    // create consumer and subscription
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Shared).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, totalProducedMessages, 0);
    List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);
    Message<byte[]> message = null;
    MessageIdImpl resetMessageId = null;
    int resetPositionId = 10;
    for (int i = 0; i < 20; i++) {
        message = consumer.receive(1, TimeUnit.SECONDS);
        consumer.acknowledge(message);
        if (i == resetPositionId) {
            resetMessageId = (MessageIdImpl) message.getMessageId();
        }
    }
    // close consumer which will clean up intenral-receive-queue
    consumer.close();
    // messages should still be available due to retention
    MessageIdImpl messageId = new MessageIdImpl(resetMessageId.getLedgerId(), resetMessageId.getEntryId(), -1);
    // reset position at resetMessageId
    admin.persistentTopics().resetCursor(topicName, "my-sub", messageId);
    consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Shared).subscribe();
    MessageIdImpl msgId2 = (MessageIdImpl) consumer.receive(1, TimeUnit.SECONDS).getMessageId();
    assertEquals(resetMessageId, msgId2);
    // start with 1 because we have already received 1 msg
    int receivedAfterReset = 1;
    for (int i = 0; i < totalProducedMessages; i++) {
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
        if (message == null) {
            break;
        }
        consumer.acknowledge(message);
        ++receivedAfterReset;
    }
    assertEquals(receivedAfterReset, totalProducedMessages - resetPositionId);
    // invalid topic name
    try {
        admin.persistentTopics().resetCursor(topicName + "invalid", "my-sub", messageId);
        fail("It should have failed due to invalid topic name");
    } catch (PulsarAdminException.NotFoundException e) {
    // Ok
    }
    // invalid cursor name
    try {
        admin.persistentTopics().resetCursor(topicName, "invalid-sub", messageId);
        fail("It should have failed due to invalid subscription name");
    } catch (PulsarAdminException.NotFoundException e) {
    // Ok
    }
    // invalid position
    try {
        messageId = new MessageIdImpl(0, 0, -1);
        admin.persistentTopics().resetCursor(topicName, "my-sub", messageId);
        fail("It should have failed due to invalid subscription name");
    } catch (PulsarAdminException.PreconditionFailedException e) {
    // Ok
    }
    consumer.close();
}
Also used : RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Message(org.apache.pulsar.client.api.Message) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) PreconditionFailedException(org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Test(org.testng.annotations.Test) MockedPulsarServiceBaseTest(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest)

Example 15 with MessageIdImpl

use of org.apache.pulsar.client.impl.MessageIdImpl in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtLeastOnceProcessingFailures.

@Test
public void testAtLeastOnceProcessingFailures() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).setClassName(TestFailureFunction.class.getName()).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            if (i % 2 == 0) {
                // all messages (i % 2 == 0) will fail to process.
                continue;
            }
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeAsync(any(Message.class));
        assertEquals(10, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // only 5 succeed messages are acknowledged
        verify(consumerInstance.getConsumer(), times(5)).acknowledgeAsync(any(Message.class));
        assertEquals(5, consumerInstance.getNumMessages());
        for (int i = 0; i < 10; i++) {
            assertEquals(i % 2 == 0, consumerInstance.containMessage(new MessageIdImpl(1L, i, 0)));
        }
    }
}
Also used : FunctionConfig(org.apache.pulsar.functions.proto.Function.FunctionConfig) Message(org.apache.pulsar.client.api.Message) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) Matchers.anyString(org.mockito.Matchers.anyString) Cleanup(lombok.Cleanup) ExecutorService(java.util.concurrent.ExecutorService) MessageId(org.apache.pulsar.client.api.MessageId) Test(org.testng.annotations.Test) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest)

Aggregations

MessageIdImpl (org.apache.pulsar.client.impl.MessageIdImpl)27 Test (org.testng.annotations.Test)18 MessageId (org.apache.pulsar.client.api.MessageId)15 Message (org.apache.pulsar.client.api.Message)11 CompletableFuture (java.util.concurrent.CompletableFuture)9 ExecutorService (java.util.concurrent.ExecutorService)9 TopicName (org.apache.pulsar.common.naming.TopicName)8 List (java.util.List)6 TimeUnit (java.util.concurrent.TimeUnit)6 Cleanup (lombok.Cleanup)6 FunctionConfig (org.apache.pulsar.functions.proto.Function.FunctionConfig)6 Matchers.anyString (org.mockito.Matchers.anyString)6 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)6 Map (java.util.Map)5 Set (java.util.Set)5 PersistentTopic (org.apache.pulsar.broker.service.persistent.PersistentTopic)5 Lists (com.google.common.collect.Lists)4 Collectors (java.util.stream.Collectors)4 ManagedLedgerImpl (org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl)4 FutureUtil (org.apache.pulsar.common.util.FutureUtil)4