Examples with KeySharedMeta org.apache.pulsar.common.api.proto.KeySharedMeta used on opensource projects

Example 11 with KeySharedMeta

use of org.apache.pulsar.common.api.proto.KeySharedMeta in project pulsar by apache.

the class PersistentStickyKeyDispatcherMultipleConsumersTest method setup.

@BeforeMethod
public void setup() throws Exception {
    configMock = mock(ServiceConfiguration.class);
    doReturn(true).when(configMock).isSubscriptionRedeliveryTrackerEnabled();
    doReturn(100).when(configMock).getDispatcherMaxReadBatchSize();
    doReturn(true).when(configMock).isSubscriptionKeySharedUseConsistentHashing();
    doReturn(1).when(configMock).getSubscriptionKeySharedConsistentHashingReplicaPoints();
    pulsarMock = mock(PulsarService.class);
    doReturn(configMock).when(pulsarMock).getConfiguration();
    brokerMock = mock(BrokerService.class);
    doReturn(pulsarMock).when(brokerMock).pulsar();
    HierarchyTopicPolicies topicPolicies = new HierarchyTopicPolicies();
    topicPolicies.getMaxConsumersPerSubscription().updateBrokerValue(0);
    EventLoopGroup eventLoopGroup = mock(EventLoopGroup.class);
    doReturn(eventLoopGroup).when(brokerMock).executor();
    doAnswer(invocation -> {
        ((Runnable) invocation.getArguments()[0]).run();
        return null;
    }).when(eventLoopGroup).execute(any(Runnable.class));
    topicMock = mock(PersistentTopic.class);
    doReturn(brokerMock).when(topicMock).getBrokerService();
    doReturn(topicName).when(topicMock).getName();
    doReturn(topicPolicies).when(topicMock).getHierarchyTopicPolicies();
    cursorMock = mock(ManagedCursorImpl.class);
    doReturn(null).when(cursorMock).getLastIndividualDeletedRange();
    doReturn(subscriptionName).when(cursorMock).getName();
    consumerMock = mock(Consumer.class);
    channelMock = mock(ChannelPromise.class);
    doReturn("consumer1").when(consumerMock).consumerName();
    doReturn(1000).when(consumerMock).getAvailablePermits();
    doReturn(true).when(consumerMock).isWritable();
    doReturn(channelMock).when(consumerMock).sendMessages(anyList(), any(EntryBatchSizes.class), any(EntryBatchIndexesAcks.class), anyInt(), anyLong(), anyLong(), any(RedeliveryTracker.class));
    subscriptionMock = mock(PersistentSubscription.class);
    try (MockedStatic<DispatchRateLimiter> rateLimiterMockedStatic = mockStatic(DispatchRateLimiter.class)) {
        rateLimiterMockedStatic.when(() -> DispatchRateLimiter.isDispatchRateNeeded(any(BrokerService.class), any(Optional.class), anyString(), any(DispatchRateLimiter.Type.class))).thenReturn(false);
        persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers(topicMock, cursorMock, subscriptionMock, configMock, new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT));
    }
}

Example 12 with KeySharedMeta

use of org.apache.pulsar.common.api.proto.KeySharedMeta in project pulsar by apache.

the class PersistentStickyKeyDispatcherMultipleConsumersTest method testSendMessage.

@Test(timeOut = 10000)
public void testSendMessage() {
    try (MockedStatic<DispatchRateLimiter> rateLimiterMockedStatic = mockStatic(DispatchRateLimiter.class)) {
        rateLimiterMockedStatic.when(() -> DispatchRateLimiter.isDispatchRateNeeded(any(BrokerService.class), any(Optional.class), anyString(), any(DispatchRateLimiter.Type.class))).thenReturn(false);
        KeySharedMeta keySharedMeta = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY);
        DispatchRateLimiter.isDispatchRateNeeded(brokerMock, Optional.empty(), "hello", DispatchRateLimiter.Type.SUBSCRIPTION);
        PersistentStickyKeyDispatcherMultipleConsumers persistentDispatcher = new PersistentStickyKeyDispatcherMultipleConsumers(topicMock, cursorMock, subscriptionMock, configMock, keySharedMeta);
        try {
            keySharedMeta.addHashRange().setStart(0).setEnd(9);
            Consumer consumerMock = mock(Consumer.class);
            doReturn(keySharedMeta).when(consumerMock).getKeySharedMeta();
            persistentDispatcher.addConsumer(consumerMock);
            persistentDispatcher.consumerFlow(consumerMock, 1000);
        } catch (Exception e) {
            fail("Failed to add mock consumer", e);
        }
        List<Entry> entries = new ArrayList<>();
        entries.add(EntryImpl.create(1, 1, createMessage("message1", 1)));
        entries.add(EntryImpl.create(1, 2, createMessage("message2", 2)));
        try {
            // Should success,see issue #8960
            persistentDispatcher.readEntriesComplete(entries, PersistentStickyKeyDispatcherMultipleConsumers.ReadType.Normal);
        } catch (Exception e) {
            fail("Failed to readEntriesComplete.", e);
        }
    }
}

Example 13 with KeySharedMeta

use of org.apache.pulsar.common.api.proto.KeySharedMeta in project pulsar by apache.

the class Commands method newSubscribe.

public static ByteBuf newSubscribe(String topic, String subscription, long consumerId, long requestId, SubType subType, int priorityLevel, String consumerName, boolean isDurable, MessageIdData startMessageId, Map<String, String> metadata, boolean readCompacted, boolean isReplicated, InitialPosition subscriptionInitialPosition, long startMessageRollbackDurationInSec, SchemaInfo schemaInfo, boolean createTopicIfDoesNotExist, KeySharedPolicy keySharedPolicy, Map<String, String> subscriptionProperties, long consumerEpoch) {
    BaseCommand cmd = localCmd(Type.SUBSCRIBE);
    CommandSubscribe subscribe = cmd.setSubscribe().setTopic(topic).setSubscription(subscription).setSubType(subType).setConsumerId(consumerId).setConsumerName(consumerName).setRequestId(requestId).setPriorityLevel(priorityLevel).setDurable(isDurable).setReadCompacted(readCompacted).setInitialPosition(subscriptionInitialPosition).setReplicateSubscriptionState(isReplicated).setForceTopicCreation(createTopicIfDoesNotExist).setConsumerEpoch(consumerEpoch);
    if (subscriptionProperties != null && !subscriptionProperties.isEmpty()) {
        List<KeyValue> keyValues = new ArrayList<>();
        subscriptionProperties.forEach((key, value) -> {
            KeyValue keyValue = new KeyValue();
            keyValue.setKey(key);
            keyValue.setValue(value);
            keyValues.add(keyValue);
        });
        subscribe.addAllSubscriptionProperties(keyValues);
    }
    if (keySharedPolicy != null) {
        KeySharedMeta keySharedMeta = subscribe.setKeySharedMeta();
        keySharedMeta.setAllowOutOfOrderDelivery(keySharedPolicy.isAllowOutOfOrderDelivery());
        keySharedMeta.setKeySharedMode(convertKeySharedMode(keySharedPolicy.getKeySharedMode()));
        if (keySharedPolicy instanceof KeySharedPolicy.KeySharedPolicySticky) {
            List<Range> ranges = ((KeySharedPolicy.KeySharedPolicySticky) keySharedPolicy).getRanges();
            for (Range range : ranges) {
                IntRange r = keySharedMeta.addHashRange();
                r.setStart(range.getStart());
                r.setEnd(range.getEnd());
            }
        }
    }
    if (startMessageId != null) {
        subscribe.setStartMessageId().copyFrom(startMessageId);
    }
    if (startMessageRollbackDurationInSec > 0) {
        subscribe.setStartMessageRollbackDurationSec(startMessageRollbackDurationInSec);
    }
    if (!metadata.isEmpty()) {
        metadata.entrySet().forEach(e -> subscribe.addMetadata().setKey(e.getKey()).setValue(e.getValue()));
    }
    if (schemaInfo != null) {
        if (subscribe.hasSchema()) {
            throw new IllegalStateException();
        }
        if (subscribe.setSchema().getPropertiesCount() > 0) {
            throw new IllegalStateException();
        }
        convertSchema(schemaInfo, subscribe.setSchema());
    }
    return serializeWithSize(cmd);
}

Example 14 with KeySharedMeta

use of org.apache.pulsar.common.api.proto.KeySharedMeta in project pulsar by apache.

the class ServerCnx method handleSubscribe.

@Override
protected void handleSubscribe(final CommandSubscribe subscribe) {
    checkArgument(state == State.Connected);
    final long requestId = subscribe.getRequestId();
    final long consumerId = subscribe.getConsumerId();
    TopicName topicName = validateTopicName(subscribe.getTopic(), requestId, subscribe);
    if (topicName == null) {
        return;
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}] Handle subscribe command: auth role = {}, original auth role = {}", remoteAddress, authRole, originalPrincipal);
    }
    if (invalidOriginalPrincipal(originalPrincipal)) {
        final String msg = "Valid Proxy Client role should be provided while subscribing ";
        log.warn("[{}] {} with role {} and proxyClientAuthRole {} on topic {}", remoteAddress, msg, authRole, originalPrincipal, topicName);
        commandSender.sendErrorResponse(requestId, ServerError.AuthorizationError, msg);
        return;
    }
    final String subscriptionName = subscribe.getSubscription();
    final SubType subType = subscribe.getSubType();
    final String consumerName = subscribe.hasConsumerName() ? subscribe.getConsumerName() : "";
    final boolean isDurable = subscribe.isDurable();
    final MessageIdImpl startMessageId = subscribe.hasStartMessageId() ? new BatchMessageIdImpl(subscribe.getStartMessageId().getLedgerId(), subscribe.getStartMessageId().getEntryId(), subscribe.getStartMessageId().getPartition(), subscribe.getStartMessageId().getBatchIndex()) : null;
    final int priorityLevel = subscribe.hasPriorityLevel() ? subscribe.getPriorityLevel() : 0;
    final boolean readCompacted = subscribe.hasReadCompacted() && subscribe.isReadCompacted();
    final Map<String, String> metadata = CommandUtils.metadataFromCommand(subscribe);
    final InitialPosition initialPosition = subscribe.getInitialPosition();
    final long startMessageRollbackDurationSec = subscribe.hasStartMessageRollbackDurationSec() ? subscribe.getStartMessageRollbackDurationSec() : -1;
    final SchemaData schema = subscribe.hasSchema() ? getSchema(subscribe.getSchema()) : null;
    final boolean isReplicated = subscribe.hasReplicateSubscriptionState() && subscribe.isReplicateSubscriptionState();
    final boolean forceTopicCreation = subscribe.isForceTopicCreation();
    final KeySharedMeta keySharedMeta = subscribe.hasKeySharedMeta() ? new KeySharedMeta().copyFrom(subscribe.getKeySharedMeta()) : emptyKeySharedMeta;
    CompletableFuture<Boolean> isAuthorizedFuture = isTopicOperationAllowed(topicName, subscriptionName, TopicOperation.CONSUME);
    // Make sure the consumer future is put into the consumers map first to avoid the same consumer
    // epoch using different consumer futures, and only remove the consumer future from the map
    // if subscribe failed .
    CompletableFuture<Consumer> consumerFuture = new CompletableFuture<>();
    CompletableFuture<Consumer> existingConsumerFuture = consumers.putIfAbsent(consumerId, consumerFuture);
    isAuthorizedFuture.thenApply(isAuthorized -> {
        if (isAuthorized) {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Client is authorized to subscribe with role {}", remoteAddress, getPrincipal());
            }
            log.info("[{}] Subscribing on topic {} / {}", remoteAddress, topicName, subscriptionName);
            try {
                Metadata.validateMetadata(metadata);
            } catch (IllegalArgumentException iae) {
                final String msg = iae.getMessage();
                consumers.remove(consumerId, consumerFuture);
                commandSender.sendErrorResponse(requestId, ServerError.MetadataError, msg);
                return null;
            }
            if (existingConsumerFuture != null) {
                if (existingConsumerFuture.isDone() && !existingConsumerFuture.isCompletedExceptionally()) {
                    Consumer consumer = existingConsumerFuture.getNow(null);
                    log.info("[{}] Consumer with the same id is already created:" + " consumerId={}, consumer={}", remoteAddress, consumerId, consumer);
                    commandSender.sendSuccessResponse(requestId);
                    return null;
                } else {
                    // There was an early request to create a consumer with same consumerId. This can happen
                    // when
                    // client timeout is lower the broker timeouts. We need to wait until the previous
                    // consumer
                    // creation request either complete or fails.
                    log.warn("[{}][{}][{}] Consumer with id is already present on the connection," + " consumerId={}", remoteAddress, topicName, subscriptionName, consumerId);
                    ServerError error = null;
                    if (!existingConsumerFuture.isDone()) {
                        error = ServerError.ServiceNotReady;
                    } else {
                        error = getErrorCode(existingConsumerFuture);
                        consumers.remove(consumerId, existingConsumerFuture);
                    }
                    commandSender.sendErrorResponse(requestId, error, "Consumer is already present on the connection");
                    return null;
                }
            }
            boolean createTopicIfDoesNotExist = forceTopicCreation && service.isAllowAutoTopicCreation(topicName.toString());
            service.getTopic(topicName.toString(), createTopicIfDoesNotExist).thenCompose(optTopic -> {
                if (!optTopic.isPresent()) {
                    return FutureUtil.failedFuture(new TopicNotFoundException("Topic " + topicName + " does not exist"));
                }
                Topic topic = optTopic.get();
                boolean rejectSubscriptionIfDoesNotExist = isDurable && !service.isAllowAutoSubscriptionCreation(topicName.toString()) && !topic.getSubscriptions().containsKey(subscriptionName) && topic.isPersistent();
                if (rejectSubscriptionIfDoesNotExist) {
                    return FutureUtil.failedFuture(new SubscriptionNotFoundException("Subscription does not exist"));
                }
                long consumerEpoch = DEFAULT_CONSUMER_EPOCH;
                if (subscribe.hasConsumerEpoch()) {
                    consumerEpoch = subscribe.getConsumerEpoch();
                }
                SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this).subscriptionName(subscriptionName).consumerId(consumerId).subType(subType).priorityLevel(priorityLevel).consumerName(consumerName).isDurable(isDurable).startMessageId(startMessageId).metadata(metadata).readCompacted(readCompacted).initialPosition(initialPosition).startMessageRollbackDurationSec(startMessageRollbackDurationSec).replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta).subscriptionProperties(SubscriptionOption.getPropertiesMap(subscribe.getSubscriptionPropertiesList())).consumerEpoch(consumerEpoch).build();
                if (schema != null) {
                    return topic.addSchemaIfIdleOrCheckCompatible(schema).thenCompose(v -> topic.subscribe(option));
                } else {
                    return topic.subscribe(option);
                }
            }).thenAccept(consumer -> {
                if (consumerFuture.complete(consumer)) {
                    log.info("[{}] Created subscription on topic {} / {}", remoteAddress, topicName, subscriptionName);
                    commandSender.sendSuccessResponse(requestId);
                    if (getBrokerService().getInterceptor() != null) {
                        getBrokerService().getInterceptor().consumerCreated(this, consumer, metadata);
                    }
                } else {
                    // The consumer future was completed before by a close command
                    try {
                        consumer.close();
                        log.info("[{}] Cleared consumer created after timeout on client side {}", remoteAddress, consumer);
                    } catch (BrokerServiceException e) {
                        log.warn("[{}] Error closing consumer created" + " after timeout on client side {}: {}", remoteAddress, consumer, e.getMessage());
                    }
                    consumers.remove(consumerId, consumerFuture);
                }
            }).exceptionally(exception -> {
                if (exception.getCause() instanceof ConsumerBusyException) {
                    if (log.isDebugEnabled()) {
                        log.debug("[{}][{}][{}] Failed to create consumer because exclusive consumer" + " is already connected: {}", remoteAddress, topicName, subscriptionName, exception.getCause().getMessage());
                    }
                } else if (exception.getCause() instanceof BrokerServiceException) {
                    log.warn("[{}][{}][{}] Failed to create consumer: consumerId={}, {}", remoteAddress, topicName, subscriptionName, consumerId, exception.getCause().getMessage());
                } else {
                    log.warn("[{}][{}][{}] Failed to create consumer: consumerId={}, {}", remoteAddress, topicName, subscriptionName, consumerId, exception.getCause().getMessage(), exception);
                }
                // back to client, only if not completed already.
                if (consumerFuture.completeExceptionally(exception)) {
                    commandSender.sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode(exception), exception.getCause().getMessage());
                }
                consumers.remove(consumerId, consumerFuture);
                return null;
            });
        } else {
            String msg = "Client is not authorized to subscribe";
            log.warn("[{}] {} with role {}", remoteAddress, msg, getPrincipal());
            consumers.remove(consumerId, consumerFuture);
            ctx.writeAndFlush(Commands.newError(requestId, ServerError.AuthorizationError, msg));
        }
        return null;
    }).exceptionally(ex -> {
        logAuthException(remoteAddress, "subscribe", getPrincipal(), Optional.of(topicName), ex);
        consumers.remove(consumerId, consumerFuture);
        commandSender.sendErrorResponse(requestId, ServerError.AuthorizationError, ex.getMessage());
        return null;
    });
}

Example 15 with KeySharedMeta

use of org.apache.pulsar.common.api.proto.KeySharedMeta in project pulsar by apache.

the class HashRangeExclusiveStickyKeyConsumerSelectorTest method testSingleRangeConflict.

@Test
public void testSingleRangeConflict() throws BrokerServiceException.ConsumerAssignException {
    HashRangeExclusiveStickyKeyConsumerSelector selector = new HashRangeExclusiveStickyKeyConsumerSelector(10);
    Consumer consumer1 = mock(Consumer.class);
    KeySharedMeta keySharedMeta1 = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY);
    keySharedMeta1.addHashRange().setStart(2).setEnd(5);
    when(consumer1.getKeySharedMeta()).thenReturn(keySharedMeta1);
    Assert.assertEquals(consumer1.getKeySharedMeta(), keySharedMeta1);
    selector.addConsumer(consumer1);
    Assert.assertEquals(selector.getRangeConsumer().size(), 2);
    final List<IntRange> testRanges = new ArrayList<>();
    testRanges.add(new IntRange().setStart(4).setEnd(6));
    testRanges.add(new IntRange().setStart(1).setEnd(3));
    testRanges.add(new IntRange().setStart(2).setEnd(2));
    testRanges.add(new IntRange().setStart(5).setEnd(5));
    testRanges.add(new IntRange().setStart(1).setEnd(5));
    testRanges.add(new IntRange().setStart(2).setEnd(6));
    testRanges.add(new IntRange().setStart(2).setEnd(5));
    testRanges.add(new IntRange().setStart(1).setEnd(6));
    testRanges.add(new IntRange().setStart(8).setEnd(6));
    for (IntRange testRange : testRanges) {
        Consumer consumer = mock(Consumer.class);
        KeySharedMeta keySharedMeta = new KeySharedMeta().setKeySharedMode(KeySharedMode.STICKY);
        keySharedMeta.addHashRange().copyFrom(testRange);
        when(consumer.getKeySharedMeta()).thenReturn(keySharedMeta);
        Assert.assertEquals(consumer.getKeySharedMeta(), keySharedMeta);
        try {
            selector.addConsumer(consumer);
            Assert.fail("should be failed");
        } catch (BrokerServiceException.ConsumerAssignException ignore) {
        }
        Assert.assertEquals(selector.getRangeConsumer().size(), 2);
    }
}