Examples with Message com.yahoo.pulsar.client.api.Message used on opensource projects

Example 56 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PulsarBoltTest method testBasic.

@Test
public void testBasic() throws Exception {
    String msgContent = "hello world";
    Tuple tuple = getMockTuple(msgContent);
    bolt.execute(tuple);
    for (int i = 0; i < NO_OF_RETRIES; i++) {
        Thread.sleep(1000);
        if (mockCollector.acked()) {
            break;
        }
    }
    Assert.assertTrue(mockCollector.acked());
    Assert.assertFalse(mockCollector.failed());
    Assert.assertNull(mockCollector.getLastError());
    Assert.assertEquals(tuple, mockCollector.getAckedTuple());
    Message msg = consumer.receive(5, TimeUnit.SECONDS);
    consumer.acknowledge(msg);
    Assert.assertEquals(msgContent, new String(msg.getData()));
}

Example 57 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PulsarBoltTest method testMetrics.

@Test
public void testMetrics() throws Exception {
    bolt.resetMetrics();
    String msgContent = "hello world";
    Tuple tuple = getMockTuple(msgContent);
    for (int i = 0; i < 10; i++) {
        bolt.execute(tuple);
    }
    for (int i = 0; i < NO_OF_RETRIES; i++) {
        Thread.sleep(1000);
        if (mockCollector.getNumTuplesAcked() == 10) {
            break;
        }
    }
    @SuppressWarnings("rawtypes") Map metrics = (Map) bolt.getValueAndReset();
    Assert.assertEquals(((Long) metrics.get(PulsarBolt.NO_OF_MESSAGES_SENT)).longValue(), 10);
    Assert.assertEquals(((Double) metrics.get(PulsarBolt.PRODUCER_RATE)).doubleValue(), 10.0 / pulsarBoltConf.getMetricsTimeIntervalInSecs());
    Assert.assertEquals(((Double) metrics.get(PulsarBolt.PRODUCER_THROUGHPUT_BYTES)).doubleValue(), ((double) msgContent.getBytes().length * 10) / pulsarBoltConf.getMetricsTimeIntervalInSecs());
    metrics = bolt.getMetrics();
    Assert.assertEquals(((Long) metrics.get(PulsarBolt.NO_OF_MESSAGES_SENT)).longValue(), 0);
    for (int i = 0; i < 10; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        consumer.acknowledge(msg);
    }
}

Example 58 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class StormExample method main.

public static void main(String[] args) throws PulsarClientException {
    ClientConfiguration clientConf = new ClientConfiguration();
    // String authPluginClassName = "com.yahoo.pulsar.client.impl.auth.MyAuthentication";
    // String authParams = "key1:val1,key2:val2";
    // clientConf.setAuthentication(authPluginClassName, authParams);
    String topic1 = "persistent://my-property/use/my-ns/my-topic1";
    String topic2 = "persistent://my-property/use/my-ns/my-topic2";
    String subscriptionName1 = "my-subscriber-name1";
    String subscriptionName2 = "my-subscriber-name2";
    // create spout
    PulsarSpoutConfiguration spoutConf = new PulsarSpoutConfiguration();
    spoutConf.setServiceUrl(serviceUrl);
    spoutConf.setTopic(topic1);
    spoutConf.setSubscriptionName(subscriptionName1);
    spoutConf.setMessageToValuesMapper(messageToValuesMapper);
    PulsarSpout spout = new PulsarSpout(spoutConf, clientConf);
    // create bolt
    PulsarBoltConfiguration boltConf = new PulsarBoltConfiguration();
    boltConf.setServiceUrl(serviceUrl);
    boltConf.setTopic(topic2);
    boltConf.setTupleToMessageMapper(tupleToMessageMapper);
    PulsarBolt bolt = new PulsarBolt(boltConf, clientConf);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("testSpout", spout);
    builder.setBolt("testBolt", bolt).shuffleGrouping("testSpout");
    Config conf = new Config();
    conf.setNumWorkers(2);
    conf.setDebug(true);
    conf.registerMetricsConsumer(PulsarMetricsConsumer.class);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    PulsarClient pulsarClient = PulsarClient.create(serviceUrl, clientConf);
    // create a consumer on topic2 to receive messages from the bolt when the processing is done
    Consumer consumer = pulsarClient.subscribe(topic2, subscriptionName2);
    // create a producer on topic1 to send messages that will be received by the spout
    Producer producer = pulsarClient.createProducer(topic1);
    for (int i = 0; i < 10; i++) {
        String msg = "msg-" + i;
        producer.send(msg.getBytes());
        LOG.info("Message {} sent", msg);
    }
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        LOG.info("Message {} received", new String(msg.getData()));
    }
    cluster.killTopology("test");
    cluster.shutdown();
}

Example 59 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PerformanceConsumer method main.

public static void main(String[] args) throws Exception {
    final Arguments arguments = new Arguments();
    JCommander jc = new JCommander(arguments);
    jc.setProgramName("pulsar-perf-consumer");
    try {
        jc.parse(args);
    } catch (ParameterException e) {
        System.out.println(e.getMessage());
        jc.usage();
        System.exit(-1);
    }
    if (arguments.help) {
        jc.usage();
        System.exit(-1);
    }
    if (arguments.topic.size() != 1) {
        System.out.println("Only one destination name is allowed");
        jc.usage();
        System.exit(-1);
    }
    if (arguments.confFile != null) {
        Properties prop = new Properties(System.getProperties());
        prop.load(new FileInputStream(arguments.confFile));
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("brokerServiceUrl");
        }
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("webServiceUrl");
        }
        // fallback to previous-version serviceUrl property to maintain backward-compatibility
        if (arguments.serviceURL == null) {
            arguments.serviceURL = prop.getProperty("serviceUrl", "http://localhost:8080/");
        }
        if (arguments.authPluginClassName == null) {
            arguments.authPluginClassName = prop.getProperty("authPlugin", null);
        }
        if (arguments.authParams == null) {
            arguments.authParams = prop.getProperty("authParams", null);
        }
    }
    // Dump config variables
    ObjectMapper m = new ObjectMapper();
    ObjectWriter w = m.writerWithDefaultPrettyPrinter();
    log.info("Starting Pulsar performance consumer with config: {}", w.writeValueAsString(arguments));
    final DestinationName prefixDestinationName = DestinationName.get(arguments.topic.get(0));
    final RateLimiter limiter = arguments.rate > 0 ? RateLimiter.create(arguments.rate) : null;
    MessageListener listener = new MessageListener() {

        public void received(Consumer consumer, Message msg) {
            messagesReceived.increment();
            bytesReceived.add(msg.getData().length);
            if (limiter != null) {
                limiter.acquire();
            }
            consumer.acknowledgeAsync(msg);
        }
    };
    EventLoopGroup eventLoopGroup;
    if (SystemUtils.IS_OS_LINUX) {
        eventLoopGroup = new EpollEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2, new DefaultThreadFactory("pulsar-perf-consumer"));
    } else {
        eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors(), new DefaultThreadFactory("pulsar-perf-consumer"));
    }
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setConnectionsPerBroker(arguments.maxConnections);
    clientConf.setStatsInterval(arguments.statsIntervalSeconds, TimeUnit.SECONDS);
    if (isNotBlank(arguments.authPluginClassName)) {
        clientConf.setAuthentication(arguments.authPluginClassName, arguments.authParams);
    }
    PulsarClient pulsarClient = new PulsarClientImpl(arguments.serviceURL, clientConf, eventLoopGroup);
    List<Future<Consumer>> futures = Lists.newArrayList();
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setMessageListener(listener);
    consumerConfig.setReceiverQueueSize(arguments.receiverQueueSize);
    for (int i = 0; i < arguments.numDestinations; i++) {
        final DestinationName destinationName = (arguments.numDestinations == 1) ? prefixDestinationName : DestinationName.get(String.format("%s-%d", prefixDestinationName, i));
        log.info("Adding {} consumers on destination {}", arguments.numConsumers, destinationName);
        for (int j = 0; j < arguments.numConsumers; j++) {
            String subscriberName;
            if (arguments.numConsumers > 1) {
                subscriberName = String.format("%s-%d", arguments.subscriberName, j);
            } else {
                subscriberName = arguments.subscriberName;
            }
            futures.add(pulsarClient.subscribeAsync(destinationName.toString(), subscriberName, consumerConfig));
        }
    }
    for (Future<Consumer> future : futures) {
        future.get();
    }
    log.info("Start receiving from {} consumers on {} destinations", arguments.numConsumers, arguments.numDestinations);
    long oldTime = System.nanoTime();
    while (true) {
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            break;
        }
        long now = System.nanoTime();
        double elapsed = (now - oldTime) / 1e9;
        double rate = messagesReceived.sumThenReset() / elapsed;
        double throughput = bytesReceived.sumThenReset() / elapsed * 8 / 1024 / 1024;
        log.info("Throughput received: {}  msg/s -- {} Mbit/s", dec.format(rate), dec.format(throughput));
        oldTime = now;
    }
    pulsarClient.close();
}

Example 60 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class ProducerHandler method onWebSocketText.

@Override
public void onWebSocketText(String message) {
    ProducerMessage sendRequest;
    byte[] rawPayload = null;
    String requestContext = null;
    try {
        sendRequest = ObjectMapperFactory.getThreadLocal().readValue(message, ProducerMessage.class);
        requestContext = sendRequest.context;
        rawPayload = Base64.getDecoder().decode(sendRequest.payload);
    } catch (IOException e) {
        sendAckResponse(new ProducerAck(FailedToDeserializeFromJSON, e.getMessage(), null, null));
        return;
    } catch (IllegalArgumentException e) {
        String msg = format("Invalid Base64 message-payload error=%s", e.getMessage());
        sendAckResponse(new ProducerAck(PayloadEncodingError, msg, null, requestContext));
        return;
    }
    MessageBuilder builder = MessageBuilder.create().setContent(rawPayload);
    if (sendRequest.properties != null) {
        builder.setProperties(sendRequest.properties);
    }
    if (sendRequest.key != null) {
        builder.setKey(sendRequest.key);
    }
    if (sendRequest.replicationClusters != null) {
        builder.setReplicationClusters(sendRequest.replicationClusters);
    }
    Message msg = builder.build();
    producer.sendAsync(msg).thenAccept(msgId -> {
        if (isConnected()) {
            String messageId = Base64.getEncoder().encodeToString(msgId.toByteArray());
            sendAckResponse(new ProducerAck(messageId, sendRequest.context));
        }
    }).exceptionally(exception -> {
        sendAckResponse(new ProducerAck(UnknownError, exception.getMessage(), null, sendRequest.context));
        return null;
    });
}