Examples with Message org.apache.pulsar.client.api.Message used on opensource projects

Example 46 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PulsarBoltTest method testBasic.

@Test
public void testBasic() throws Exception {
    String msgContent = "hello world";
    Tuple tuple = getMockTuple(msgContent);
    bolt.execute(tuple);
    for (int i = 0; i < NO_OF_RETRIES; i++) {
        Thread.sleep(1000);
        if (mockCollector.acked()) {
            break;
        }
    }
    Assert.assertTrue(mockCollector.acked());
    Assert.assertFalse(mockCollector.failed());
    Assert.assertNull(mockCollector.getLastError());
    Assert.assertEquals(tuple, mockCollector.getAckedTuple());
    Message msg = consumer.receive(5, TimeUnit.SECONDS);
    consumer.acknowledge(msg);
    Assert.assertEquals(msgContent, new String(msg.getData()));
}

Example 47 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class StormExample method main.

public static void main(String[] args) throws PulsarClientException {
    ClientConfiguration clientConf = new ClientConfiguration();
    // String authPluginClassName = "org.apache.pulsar.client.impl.auth.MyAuthentication";
    // String authParams = "key1:val1,key2:val2";
    // clientConf.setAuthentication(authPluginClassName, authParams);
    String topic1 = "persistent://my-property/use/my-ns/my-topic1";
    String topic2 = "persistent://my-property/use/my-ns/my-topic2";
    String subscriptionName1 = "my-subscriber-name1";
    String subscriptionName2 = "my-subscriber-name2";
    // create spout
    PulsarSpoutConfiguration spoutConf = new PulsarSpoutConfiguration();
    spoutConf.setServiceUrl(serviceUrl);
    spoutConf.setTopic(topic1);
    spoutConf.setSubscriptionName(subscriptionName1);
    spoutConf.setMessageToValuesMapper(messageToValuesMapper);
    PulsarSpout spout = new PulsarSpout(spoutConf, clientConf);
    // create bolt
    PulsarBoltConfiguration boltConf = new PulsarBoltConfiguration();
    boltConf.setServiceUrl(serviceUrl);
    boltConf.setTopic(topic2);
    boltConf.setTupleToMessageMapper(tupleToMessageMapper);
    PulsarBolt bolt = new PulsarBolt(boltConf, clientConf);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("testSpout", spout);
    builder.setBolt("testBolt", bolt).shuffleGrouping("testSpout");
    Config conf = new Config();
    conf.setNumWorkers(2);
    conf.setDebug(true);
    conf.registerMetricsConsumer(PulsarMetricsConsumer.class);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    PulsarClient pulsarClient = PulsarClient.create(serviceUrl, clientConf);
    // create a consumer on topic2 to receive messages from the bolt when the processing is done
    Consumer consumer = pulsarClient.subscribe(topic2, subscriptionName2);
    // create a producer on topic1 to send messages that will be received by the spout
    Producer producer = pulsarClient.createProducer(topic1);
    for (int i = 0; i < 10; i++) {
        String msg = "msg-" + i;
        producer.send(msg.getBytes());
        LOG.info("Message {} sent", msg);
    }
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        LOG.info("Message {} received", new String(msg.getData()));
    }
    cluster.killTopology("test");
    cluster.shutdown();
}

Example 48 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class ProducerHandler method onWebSocketText.

@Override
public void onWebSocketText(String message) {
    ProducerMessage sendRequest;
    byte[] rawPayload = null;
    String requestContext = null;
    try {
        sendRequest = ObjectMapperFactory.getThreadLocal().readValue(message, ProducerMessage.class);
        requestContext = sendRequest.context;
        rawPayload = Base64.getDecoder().decode(sendRequest.payload);
    } catch (IOException e) {
        sendAckResponse(new ProducerAck(FailedToDeserializeFromJSON, e.getMessage(), null, null));
        return;
    } catch (IllegalArgumentException e) {
        String msg = format("Invalid Base64 message-payload error=%s", e.getMessage());
        sendAckResponse(new ProducerAck(PayloadEncodingError, msg, null, requestContext));
        return;
    }
    final long msgSize = rawPayload.length;
    MessageBuilder builder = MessageBuilder.create().setContent(rawPayload);
    if (sendRequest.properties != null) {
        builder.setProperties(sendRequest.properties);
    }
    if (sendRequest.key != null) {
        builder.setKey(sendRequest.key);
    }
    if (sendRequest.replicationClusters != null) {
        builder.setReplicationClusters(sendRequest.replicationClusters);
    }
    Message<byte[]> msg = builder.build();
    final long now = System.nanoTime();
    producer.sendAsync(msg).thenAccept(msgId -> {
        updateSentMsgStats(msgSize, TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - now));
        if (isConnected()) {
            String messageId = Base64.getEncoder().encodeToString(msgId.toByteArray());
            sendAckResponse(new ProducerAck(messageId, sendRequest.context));
        }
    }).exceptionally(exception -> {
        numMsgsFailed.increment();
        sendAckResponse(new ProducerAck(UnknownError, exception.getMessage(), null, sendRequest.context));
        return null;
    });
}

Example 49 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PulsarKafkaConsumer method poll.

@SuppressWarnings("unchecked")
@Override
public ConsumerRecords<K, V> poll(long timeoutMillis) {
    try {
        QueueItem item = receivedMessages.poll(timeoutMillis, TimeUnit.MILLISECONDS);
        if (item == null) {
            return (ConsumerRecords<K, V>) ConsumerRecords.EMPTY;
        }
        Map<TopicPartition, List<ConsumerRecord<K, V>>> records = new HashMap<>();
        int numberOfRecords = 0;
        while (item != null && ++numberOfRecords < MAX_RECORDS_IN_SINGLE_POLL) {
            TopicName topicName = TopicName.get(item.consumer.getTopic());
            String topic = topicName.getPartitionedTopicName();
            int partition = topicName.isPartitioned() ? topicName.getPartitionIndex() : 0;
            Message<byte[]> msg = item.message;
            MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
            long offset = MessageIdUtils.getOffset(msgId);
            TopicPartition tp = new TopicPartition(topic, partition);
            K key = getKey(topic, msg);
            V value = valueDeserializer.deserialize(topic, msg.getData());
            TimestampType timestampType = TimestampType.LOG_APPEND_TIME;
            long timestamp = msg.getPublishTime();
            if (msg.getEventTime() > 0) {
                // If we have Event time, use that in preference
                timestamp = msg.getEventTime();
                timestampType = TimestampType.CREATE_TIME;
            }
            ConsumerRecord<K, V> consumerRecord = new ConsumerRecord<>(topic, partition, offset, timestamp, timestampType, -1, msg.hasKey() ? msg.getKey().length() : 0, msg.getData().length, key, value);
            records.computeIfAbsent(tp, k -> new ArrayList<>()).add(consumerRecord);
            // Update last offset seen by application
            lastReceivedOffset.put(tp, offset);
            // Check if we have an item already available
            item = receivedMessages.poll(0, TimeUnit.MILLISECONDS);
        }
        if (isAutoCommit) {
            // Commit the offset of previously dequeued messages
            commitAsync();
        }
        return new ConsumerRecords<>(records);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
}

Example 50 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PulsarKafkaProducer method send.

@Override
public Future<RecordMetadata> send(ProducerRecord<K, V> record, Callback callback) {
    org.apache.pulsar.client.api.Producer<byte[]> producer;
    try {
        producer = producers.computeIfAbsent(record.topic(), topic -> createNewProducer(topic));
    } catch (Exception e) {
        if (callback != null) {
            callback.onCompletion(null, e);
        }
        CompletableFuture<RecordMetadata> future = new CompletableFuture<>();
        future.completeExceptionally(e);
        return future;
    }
    Message<byte[]> msg = getMessage(record);
    int messageSize = msg.getData().length;
    CompletableFuture<RecordMetadata> future = new CompletableFuture<>();
    CompletableFuture<MessageId> sendFuture = producer.sendAsync(msg);
    lastSendFuture.put(record.topic(), sendFuture);
    sendFuture.thenAccept((messageId) -> {
        future.complete(getRecordMetadata(record.topic(), msg, messageId, messageSize));
    }).exceptionally(ex -> {
        future.completeExceptionally(ex);
        return null;
    });
    future.handle((recordMetadata, throwable) -> {
        if (callback != null) {
            Exception exception = throwable != null ? new Exception(throwable) : null;
            callback.onCompletion(recordMetadata, exception);
        }
        return null;
    });
    return future;
}