Examples with TypedMessageBuilder org.apache.pulsar.client.api.TypedMessageBuilder used on opensource projects

Example 11 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project pulsar by yahoo.

the class CmdProduce method publish.

private int publish(String topic) {
    int numMessagesSent = 0;
    int returnCode = 0;
    try {
        PulsarClient client = clientBuilder.build();
        Schema<?> schema = buildSchema(this.keySchema, this.valueSchema, this.keyValueEncodingType);
        ProducerBuilder<?> producerBuilder = client.newProducer(schema).topic(topic);
        if (this.chunkingAllowed) {
            producerBuilder.enableChunking(true);
            producerBuilder.enableBatching(false);
        } else if (this.disableBatching) {
            producerBuilder.enableBatching(false);
        }
        if (isNotBlank(this.encKeyName) && isNotBlank(this.encKeyValue)) {
            producerBuilder.addEncryptionKey(this.encKeyName);
            producerBuilder.defaultCryptoKeyReader(this.encKeyValue);
        }
        Producer<?> producer = producerBuilder.create();
        List<byte[]> messageBodies = generateMessageBodies(this.messages, this.messageFileNames);
        RateLimiter limiter = (this.publishRate > 0) ? RateLimiter.create(this.publishRate) : null;
        Map<String, String> kvMap = new HashMap<>();
        for (String property : properties) {
            String[] kv = property.split("=");
            kvMap.put(kv[0], kv[1]);
        }
        for (int i = 0; i < this.numTimesProduce; i++) {
            for (byte[] content : messageBodies) {
                if (limiter != null) {
                    limiter.acquire();
                }
                TypedMessageBuilder message = producer.newMessage();
                if (!kvMap.isEmpty()) {
                    message.properties(kvMap);
                }
                switch(keyValueEncodingType) {
                    case KEY_VALUE_ENCODING_TYPE_NOT_SET:
                        if (key != null && !key.isEmpty()) {
                            message.key(key);
                        }
                        message.value(content);
                        break;
                    case KEY_VALUE_ENCODING_TYPE_SEPARATED:
                    case KEY_VALUE_ENCODING_TYPE_INLINE:
                        KeyValue kv = new KeyValue<>(// TODO: support AVRO encoded key
                        key != null ? key.getBytes(StandardCharsets.UTF_8) : null, content);
                        message.value(kv);
                        break;
                    default:
                        throw new IllegalStateException();
                }
                if (disableReplication) {
                    message.disableReplication();
                }
                message.send();
                numMessagesSent++;
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while producing messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully produced", numMessagesSent);
    }
    return returnCode;
}

Example 12 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project pulsar by yahoo.

the class ProducerHandlerTest method testProduceMessageAttributes.

@Test
public void testProduceMessageAttributes() throws IOException {
    String producerV2 = "/ws/v2/producer/persistent/my-property/my-ns/my-topic";
    HttpServletRequest httpServletRequest = mock(HttpServletRequest.class);
    PulsarClient pulsarClient = mock(PulsarClient.class);
    ProducerBuilder producerBuilder = mock(ProducerBuilder.class);
    Producer producer = mock(Producer.class);
    TypedMessageBuilder messageBuilder = mock(TypedMessageBuilder.class);
    ProducerMessage produceRequest = new ProducerMessage();
    produceRequest.setDeliverAfterMs(11111);
    produceRequest.setDeliverAt(22222);
    produceRequest.setContext("context");
    produceRequest.setPayload(Base64.getEncoder().encodeToString("my payload".getBytes()));
    // the params are all different with the default value
    Map<String, String[]> queryParams = new HashMap<>();
    httpServletRequest = mock(HttpServletRequest.class);
    when(httpServletRequest.getRequestURI()).thenReturn(producerV2);
    when(httpServletRequest.getParameterMap()).thenReturn(queryParams);
    WebSocketService service = mock(WebSocketService.class);
    when(service.isAuthenticationEnabled()).thenReturn(false);
    when(service.isAuthorizationEnabled()).thenReturn(false);
    when(service.getPulsarClient()).thenReturn(pulsarClient);
    when(pulsarClient.newProducer()).thenReturn(producerBuilder);
    when(producerBuilder.enableBatching(anyBoolean())).thenReturn(producerBuilder);
    when(producerBuilder.messageRoutingMode(any())).thenReturn(producerBuilder);
    when(producerBuilder.blockIfQueueFull(anyBoolean())).thenReturn(producerBuilder);
    when(producerBuilder.topic(anyString())).thenReturn(producerBuilder);
    when(producerBuilder.create()).thenReturn(producer);
    when(producer.newMessage()).thenReturn(messageBuilder);
    when(messageBuilder.sendAsync()).thenReturn(CompletableFuture.completedFuture(new MessageIdImpl(1, 2, 3)));
    ServletUpgradeResponse response = mock(ServletUpgradeResponse.class);
    ProducerHandler producerHandler = new ProducerHandler(service, httpServletRequest, response);
    producerHandler.onWebSocketText(ObjectMapperFactory.getThreadLocal().writeValueAsString(produceRequest));
    verify(messageBuilder, times(1)).deliverAfter(11111, TimeUnit.MILLISECONDS);
    verify(messageBuilder, times(1)).deliverAt(22222);
}

Example 13 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project pulsar by yahoo.

the class WindowFunctionExecutorTest method testExecuteWithLateTupleStream.

@Test
public void testExecuteWithLateTupleStream() throws Exception {
    windowConfig.setLateDataTopic("$late");
    doReturn(Optional.of(new Gson().fromJson(new Gson().toJson(windowConfig), Map.class))).when(context).getUserConfigValue(WindowConfig.WINDOW_CONFIG_KEY);
    TypedMessageBuilder typedMessageBuilder = mock(TypedMessageBuilder.class);
    when(typedMessageBuilder.value(any())).thenReturn(typedMessageBuilder);
    when(typedMessageBuilder.sendAsync()).thenReturn(CompletableFuture.anyOf());
    when(context.newOutputMessage(anyString(), any())).thenReturn(typedMessageBuilder);
    long[] timestamps = { 603, 605, 607, 618, 626, 636, 600 };
    List<Long> events = new ArrayList<>(timestamps.length);
    for (long ts : timestamps) {
        events.add(ts);
        Record<?> record = mock(Record.class);
        doReturn(Optional.of("test-topic")).when(record).getTopicName();
        doReturn(record).when(context).getCurrentRecord();
        doReturn(ts).when(record).getValue();
        testWindowedPulsarFunction.process(ts, context);
        // Update the watermark to this timestamp
        testWindowedPulsarFunction.waterMarkEventGenerator.run();
    }
    System.out.println(testWindowedPulsarFunction.windows);
    long event = events.get(events.size() - 1);
}

Example 14 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project pulsar by yahoo.

the class TypedMessageBuilderPublish method process.

@Override
public Void process(String input, Context context) {
    String publishTopic = (String) context.getUserConfigValueOrDefault("publish-topic", "publishtopic");
    String output = String.format("%s!", input);
    Map<String, String> properties = new HashMap<>();
    properties.put("input_topic", context.getCurrentRecord().getTopicName().get());
    properties.putAll(context.getCurrentRecord().getProperties());
    try {
        TypedMessageBuilder messageBuilder = context.newOutputMessage(publishTopic, Schema.STRING).value(output).properties(properties);
        if (context.getCurrentRecord().getKey().isPresent()) {
            messageBuilder.key(context.getCurrentRecord().getKey().get());
        }
        messageBuilder.eventTime(System.currentTimeMillis()).sendAsync();
    } catch (PulsarClientException e) {
        context.getLogger().error(e.toString());
    }
    return null;
}

Example 15 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project debezium by debezium.

the class PulsarChangeConsumer method handleBatch.

@SuppressWarnings("unchecked")
@Override
public void handleBatch(List<ChangeEvent<Object, Object>> records, RecordCommitter<ChangeEvent<Object, Object>> committer) throws InterruptedException {
    for (ChangeEvent<Object, Object> record : records) {
        LOGGER.trace("Received event '{}'", record);
        final String topicName = streamNameMapper.map(record.destination());
        final Producer<?> producer = producers.computeIfAbsent(topicName, (topic) -> createProducer(topic, record.value()));
        final String key = (record.key()) == null ? nullKey : getString(record.key());
        @SuppressWarnings("rawtypes") final TypedMessageBuilder message;
        if (record.value() instanceof String) {
            message = producer.newMessage(Schema.STRING);
        } else {
            message = producer.newMessage();
        }
        message.key(key).value(record.value());
        try {
            final MessageId messageId = message.send();
            LOGGER.trace("Sent message with id: {}", messageId);
        } catch (PulsarClientException e) {
            throw new DebeziumException(e);
        }
        committer.markProcessed(record);
    }
    committer.markBatchFinished();
}