Examples with ConsumerConfiguration com.yahoo.pulsar.client.api.ConsumerConfiguration used on opensource projects

Example 41 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class ClientErrorsTest method subscribeFailWithoutRetry.

private void subscribeFailWithoutRetry(String topic) throws Exception {
    PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
    final AtomicInteger counter = new AtomicInteger(0);
    mockBrokerService.setHandleSubscribe((ctx, subscribe) -> {
        if (counter.incrementAndGet() == 2) {
            // piggyback unknown error to relay assertion failure
            ctx.writeAndFlush(Commands.newError(subscribe.getRequestId(), ServerError.UnknownError, ASSERTION_ERROR));
            return;
        }
        ctx.writeAndFlush(Commands.newError(subscribe.getRequestId(), ServerError.PersistenceError, "msg"));
    });
    try {
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setSubscriptionType(SubscriptionType.Exclusive);
        Consumer consumer = client.subscribe(topic, "sub1", conf);
    } catch (Exception e) {
        if (e.getMessage().equals(ASSERTION_ERROR)) {
            fail("Subscribe should not retry on persistence error");
        }
        assertTrue(e instanceof PulsarClientException.BrokerPersistenceException);
    }
    mockBrokerService.resetHandleSubscribe();
    client.close();
}

Example 42 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class ClientErrorsTest method testLookupWithDisconnection.

@Test
public void testLookupWithDisconnection() throws Exception {
    final String brokerUrl = "pulsar://127.0.0.1:" + BROKER_SERVICE_PORT;
    PulsarClient client = PulsarClient.create(brokerUrl);
    final AtomicInteger counter = new AtomicInteger(0);
    String topic = "persistent://prop/use/ns/t1";
    mockBrokerService.setHandlePartitionLookup((ctx, lookup) -> {
        ctx.writeAndFlush(Commands.newPartitionMetadataResponse(0, lookup.getRequestId()));
    });
    mockBrokerService.setHandleLookup((ctx, lookup) -> {
        if (counter.incrementAndGet() == 1) {
            // piggyback unknown error to relay assertion failure
            ctx.close();
            return;
        }
        ctx.writeAndFlush(Commands.newLookupResponse(brokerUrl, null, true, LookupType.Connect, lookup.getRequestId()));
    });
    try {
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setSubscriptionType(SubscriptionType.Exclusive);
        Consumer consumer = client.subscribe(topic, "sub1", conf);
    } catch (Exception e) {
        if (e.getMessage().equals(ASSERTION_ERROR)) {
            fail("Subscribe should not retry on persistence error");
        }
        assertTrue(e instanceof PulsarClientException.BrokerPersistenceException);
    }
    mockBrokerService.resetHandlePartitionLookup();
    mockBrokerService.resetHandleLookup();
    client.close();
}

Example 43 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class PulsarSpoutTest method setup.

@Override
protected void setup() throws Exception {
    super.internalSetup();
    super.producerBaseSetup();
    pulsarSpoutConf = new PulsarSpoutConfiguration();
    pulsarSpoutConf.setServiceUrl(serviceUrl);
    pulsarSpoutConf.setTopic(topic);
    pulsarSpoutConf.setSubscriptionName(subscriptionName);
    pulsarSpoutConf.setMessageToValuesMapper(messageToValuesMapper);
    pulsarSpoutConf.setFailedRetriesTimeout(1, TimeUnit.SECONDS);
    pulsarSpoutConf.setMaxFailedRetries(2);
    pulsarSpoutConf.setSharedConsumerEnabled(true);
    pulsarSpoutConf.setMetricsTimeIntervalInSecs(60);
    consumerConf = new ConsumerConfiguration();
    consumerConf.setSubscriptionType(SubscriptionType.Shared);
    spout = new PulsarSpout(pulsarSpoutConf, new ClientConfiguration(), consumerConf);
    mockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(mockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(0);
    spout.open(Maps.newHashMap(), context, collector);
    producer = pulsarClient.createProducer(topic);
}

Example 44 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class BrokerClientIntegrationTest method testUnsupportedBatchMessageConsumer.

/**
     * It verifies that consumer which doesn't support batch-message:
     * <p>
     * 1. broker disconnects that consumer
     * <p>
     * 2. redeliver all those messages to other supported consumer under the same subscription
     * 
     * @param subType
     * @throws Exception
     */
@Test(timeOut = 7000, dataProvider = "subType")
public void testUnsupportedBatchMessageConsumer(SubscriptionType subType) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int batchMessageDelayMs = 1000;
    final String topicName = "persistent://my-property/use/my-ns/my-topic1";
    final String subscriptionName = "my-subscriber-name" + subType;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(subType);
    ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(20);
    }
    Producer producer = pulsarClient.createProducer(topicName, new ProducerConfiguration());
    Producer batchProducer = pulsarClient.createProducer(topicName, producerConf);
    // update consumer's version to incompatible batch-message version = Version.V3
    Topic topic = pulsar.getBrokerService().getTopic(topicName).get();
    com.yahoo.pulsar.broker.service.Consumer brokerConsumer = topic.getSubscriptions().get(subscriptionName).getConsumers().get(0);
    Field cnxField = com.yahoo.pulsar.broker.service.Consumer.class.getDeclaredField("cnx");
    cnxField.setAccessible(true);
    PulsarHandler cnx = (PulsarHandler) cnxField.get(brokerConsumer);
    Field versionField = PulsarHandler.class.getDeclaredField("remoteEndpointProtocolVersion");
    versionField.setAccessible(true);
    versionField.set(cnx, 3);
    // (1) send non-batch message: consumer should be able to consume
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Set<String> messageSet = Sets.newHashSet();
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        consumer1.acknowledge(msg);
    }
    // Also set clientCnx of the consumer to null so, it avoid reconnection so, other consumer can consume for
    // verification
    consumer1.setClientCnx(null);
    // (2) send batch-message which should not be able to consume: as broker will disconnect the consumer
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        batchProducer.sendAsync(message.getBytes());
    }
    Thread.sleep(batchMessageDelayMs);
    // consumer should have not received any message as it should have been disconnected
    msg = consumer1.receive(2, TimeUnit.SECONDS);
    assertNull(msg);
    // subscrie consumer2 with supporting batch version
    pulsarClient = PulsarClient.create(brokerUrl.toString());
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    messageSet.clear();
    for (int i = 0; i < 10; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        consumer2.acknowledge(msg);
    }
    consumer2.close();
    producer.close();
    batchProducer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 45 with ConsumerConfiguration

use of com.yahoo.pulsar.client.api.ConsumerConfiguration in project pulsar by yahoo.

the class BrokerClientIntegrationTest method testDisconnectClientWithoutClosingConnection.

/**
     * Verifies unload namespace-bundle doesn't close shared connection used by other namespace-bundle.
     * <pre>
     * 1. after disabling broker fron loadbalancer
     * 2. unload namespace-bundle "my-ns1" which disconnects client (producer/consumer) connected on that namespacebundle
     * 3. but doesn't close the connection for namesapce-bundle "my-ns2" and clients are still connected
     * 4. verifies unloaded "my-ns1" should not connected again with the broker as broker is disabled
     * 5. unload "my-ns2" which closes the connection as broker doesn't have any more client connected on that connection
     * 6. all namespace-bundles are in "connecting" state and waiting for available broker
     * </pre>
     * 
     * @throws Exception
     */
@Test
public void testDisconnectClientWithoutClosingConnection() throws Exception {
    final String ns1 = "my-property/use/con-ns1";
    final String ns2 = "my-property/use/con-ns2";
    admin.namespaces().createNamespace(ns1);
    admin.namespaces().createNamespace(ns2);
    final String dn1 = "persistent://" + ns1 + "/my-topic";
    final String dn2 = "persistent://" + ns2 + "/my-topic";
    ConsumerImpl cons1 = (ConsumerImpl) pulsarClient.subscribe(dn1, "my-subscriber-name", new ConsumerConfiguration());
    ProducerImpl prod1 = (ProducerImpl) pulsarClient.createProducer(dn1, new ProducerConfiguration());
    ProducerImpl prod2 = (ProducerImpl) pulsarClient.createProducer(dn2, new ProducerConfiguration());
    ConsumerImpl consumer1 = spy(cons1);
    doAnswer(invocationOnMock -> cons1.getState()).when(consumer1).getState();
    doAnswer(invocationOnMock -> cons1.getClientCnx()).when(consumer1).getClientCnx();
    doAnswer(invocationOnMock -> cons1.cnx()).when(consumer1).cnx();
    doAnswer(invocationOnMock -> {
        cons1.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]);
        return null;
    }).when(consumer1).connectionClosed(anyObject());
    ProducerImpl producer1 = spy(prod1);
    doAnswer(invocationOnMock -> prod1.getState()).when(producer1).getState();
    doAnswer(invocationOnMock -> prod1.getClientCnx()).when(producer1).getClientCnx();
    doAnswer(invocationOnMock -> prod1.cnx()).when(producer1).cnx();
    doAnswer(invocationOnMock -> {
        prod1.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]);
        return null;
    }).when(producer1).connectionClosed(anyObject());
    ProducerImpl producer2 = spy(prod2);
    doAnswer(invocationOnMock -> prod2.getState()).when(producer2).getState();
    doAnswer(invocationOnMock -> prod2.getClientCnx()).when(producer2).getClientCnx();
    doAnswer(invocationOnMock -> prod2.cnx()).when(producer2).cnx();
    doAnswer(invocationOnMock -> {
        prod2.connectionClosed((ClientCnx) invocationOnMock.getArguments()[0]);
        return null;
    }).when(producer2).connectionClosed(anyObject());
    ClientCnx clientCnx = producer1.getClientCnx();
    Field pfield = ClientCnx.class.getDeclaredField("producers");
    pfield.setAccessible(true);
    Field cfield = ClientCnx.class.getDeclaredField("consumers");
    cfield.setAccessible(true);
    ConcurrentLongHashMap<ProducerImpl> producers = (ConcurrentLongHashMap<ProducerImpl>) pfield.get(clientCnx);
    ConcurrentLongHashMap<ConsumerImpl> consumers = (ConcurrentLongHashMap<ConsumerImpl>) cfield.get(clientCnx);
    producers.put(2, producers.get(0));
    producers.put(3, producers.get(1));
    consumers.put(1, consumers.get(0));
    producers.put(0, producer1);
    producers.put(1, producer2);
    consumers.put(0, consumer1);
    // disable this broker to avoid any new requests
    pulsar.getLoadManager().disableBroker();
    NamespaceBundle bundle1 = pulsar.getNamespaceService().getBundle(DestinationName.get(dn1));
    NamespaceBundle bundle2 = pulsar.getNamespaceService().getBundle(DestinationName.get(dn2));
    // unload ns-bundle:1
    pulsar.getNamespaceService().unloadNamespaceBundle((NamespaceBundle) bundle1);
    // let server send signal to close-connection and client close the connection
    Thread.sleep(1000);
    // [1] Verify: producer1 must get connectionClosed signal
    verify(producer1, atLeastOnce()).connectionClosed(anyObject());
    // [2] Verify: consumer1 must get connectionClosed signal
    verify(consumer1, atLeastOnce()).connectionClosed(anyObject());
    // [3] Verify: producer2 should have not received connectionClosed signal
    verify(producer2, never()).connectionClosed(anyObject());
    // sleep for sometime to let other disconnected producer and consumer connect again: but they should not get
    // connected with same broker as that broker is already out from active-broker list
    Thread.sleep(200);
    // producer1 must not be able to connect again
    assertTrue(prod1.getClientCnx() == null);
    assertTrue(prod1.getState().equals(State.Connecting));
    // consumer1 must not be able to connect again
    assertTrue(cons1.getClientCnx() == null);
    assertTrue(cons1.getState().equals(State.Connecting));
    // producer2 must have live connection
    assertTrue(prod2.getClientCnx() != null);
    assertTrue(prod2.getState().equals(State.Ready));
    // unload ns-bundle2 as well
    pulsar.getNamespaceService().unloadNamespaceBundle((NamespaceBundle) bundle2);
    verify(producer2, atLeastOnce()).connectionClosed(anyObject());
    Thread.sleep(200);
    // producer1 must not be able to connect again
    assertTrue(prod1.getClientCnx() == null);
    assertTrue(prod1.getState().equals(State.Connecting));
    // consumer1 must not be able to connect again
    assertTrue(cons1.getClientCnx() == null);
    assertTrue(cons1.getState().equals(State.Connecting));
    // producer2 must not be able to connect again
    assertTrue(prod2.getClientCnx() == null);
    assertTrue(prod2.getState().equals(State.Connecting));
    producer1.close();
    producer2.close();
    consumer1.close();
    prod1.close();
    prod2.close();
    cons1.close();
}