Examples with TypedMessageBuilder org.apache.pulsar.client.api.TypedMessageBuilder used on opensource projects

Example 1 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project pulsar by apache.

the class SchemaTest method generateDataByDifferentSchema.

private void generateDataByDifferentSchema(String ns, String baseTopic, Schema schema, Object data, Producer<?> autoProducer, AtomicInteger totalMsgCnt) throws PulsarClientException {
    String topic = getTopicName(ns, baseTopic);
    Producer<Object> producer = pulsarClient.newProducer(schema).topic(topic).create();
    producer.newMessage().value(data).property("baseTopic", baseTopic).send();
    totalMsgCnt.incrementAndGet();
    Consumer<GenericRecord> consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()).topic(topic).subscriptionName("test").subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
    Message<GenericRecord> message = consumer.receive(5, TimeUnit.SECONDS);
    if (message == null) {
        Assert.fail("Failed to receive message for topic " + topic);
    }
    if (!message.getReaderSchema().isPresent()) {
        Assert.fail("Failed to get reader schema for topic " + topic);
    }
    message.getValue();
    TypedMessageBuilder messageBuilder = autoProducer.newMessage(Schema.AUTO_PRODUCE_BYTES(message.getReaderSchema().get())).value(message.getData()).properties(message.getProperties());
    if (message.getKeyBytes() != null) {
        messageBuilder.keyBytes(message.getKeyBytes());
    }
    messageBuilder.send();
    producer.close();
    consumer.close();
}

Example 2 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project pulsar by apache.

the class FunctionMetaDataManager method updateFunctionOnLeader.

/**
 * Called by the worker when we are in the leader mode.  In this state, we update our in-memory
 * data structures and then write to the metadata topic.
 * @param functionMetaData The function metadata in question
 * @param delete Is this a delete operation
 * @throws IllegalStateException if we are not the leader
 * @throws IllegalArgumentException if the request is out of date.
 */
public synchronized void updateFunctionOnLeader(FunctionMetaData functionMetaData, boolean delete) throws IllegalStateException, IllegalArgumentException {
    boolean needsScheduling;
    if (exclusiveLeaderProducer == null) {
        throw new IllegalStateException("Not the leader");
    }
    // Check first to avoid local cache update failure
    checkRequestOutDated(functionMetaData, delete);
    byte[] toWrite;
    if (workerConfig.getUseCompactedMetadataTopic()) {
        if (delete) {
            toWrite = "".getBytes();
        } else {
            toWrite = functionMetaData.toByteArray();
        }
    } else {
        Request.ServiceRequest serviceRequest = Request.ServiceRequest.newBuilder().setServiceRequestType(delete ? Request.ServiceRequest.ServiceRequestType.DELETE : Request.ServiceRequest.ServiceRequestType.UPDATE).setFunctionMetaData(functionMetaData).setWorkerId(workerConfig.getWorkerId()).setRequestId(UUID.randomUUID().toString()).build();
        toWrite = serviceRequest.toByteArray();
    }
    try {
        TypedMessageBuilder builder = exclusiveLeaderProducer.newMessage().value(toWrite).property(versionTag, Long.toString(functionMetaData.getVersion()));
        if (workerConfig.getUseCompactedMetadataTopic()) {
            builder = builder.key(FunctionCommon.getFullyQualifiedName(functionMetaData.getFunctionDetails()));
        }
        lastMessageSeen = builder.send();
        if (delete) {
            needsScheduling = processDeregister(functionMetaData);
        } else {
            needsScheduling = processUpdate(functionMetaData);
        }
    } catch (Exception e) {
        log.error("Could not write into Function Metadata topic", e);
        throw new IllegalStateException("Internal Error updating function at the leader", e);
    }
    if (needsScheduling) {
        this.schedulerManager.schedule();
    }
}

Example 3 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project pulsar by apache.

the class ProducerHandler method onWebSocketText.

@Override
public void onWebSocketText(String message) {
    if (log.isDebugEnabled()) {
        log.debug("[{}] Received new message from producer {} ", producer.getTopic(), getRemote().getInetSocketAddress().toString());
    }
    ProducerMessage sendRequest;
    byte[] rawPayload = null;
    String requestContext = null;
    try {
        sendRequest = ObjectMapperFactory.getThreadLocal().readValue(message, ProducerMessage.class);
        requestContext = sendRequest.context;
        rawPayload = Base64.getDecoder().decode(sendRequest.payload);
    } catch (IOException e) {
        sendAckResponse(new ProducerAck(FailedToDeserializeFromJSON, e.getMessage(), null, null));
        return;
    } catch (IllegalArgumentException e) {
        String msg = format("Invalid Base64 message-payload error=%s", e.getMessage());
        sendAckResponse(new ProducerAck(PayloadEncodingError, msg, null, requestContext));
        return;
    } catch (NullPointerException e) {
        // Null payload
        sendAckResponse(new ProducerAck(PayloadEncodingError, e.getMessage(), null, requestContext));
        return;
    }
    final long msgSize = rawPayload.length;
    TypedMessageBuilder<byte[]> builder = producer.newMessage();
    try {
        builder.value(rawPayload);
    } catch (SchemaSerializationException e) {
        sendAckResponse(new ProducerAck(PayloadEncodingError, e.getMessage(), null, requestContext));
        return;
    }
    if (sendRequest.properties != null) {
        builder.properties(sendRequest.properties);
    }
    if (sendRequest.key != null) {
        builder.key(sendRequest.key);
    }
    if (sendRequest.replicationClusters != null) {
        builder.replicationClusters(sendRequest.replicationClusters);
    }
    if (sendRequest.eventTime != null) {
        try {
            builder.eventTime(DateFormatter.parse(sendRequest.eventTime));
        } catch (DateTimeParseException e) {
            sendAckResponse(new ProducerAck(PayloadEncodingError, e.getMessage(), null, requestContext));
            return;
        }
    }
    if (sendRequest.deliverAt > 0) {
        builder.deliverAt(sendRequest.deliverAt);
    }
    if (sendRequest.deliverAfterMs > 0) {
        builder.deliverAfter(sendRequest.deliverAfterMs, TimeUnit.MILLISECONDS);
    }
    final long now = System.nanoTime();
    builder.sendAsync().thenAccept(msgId -> {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Success fully write the message to broker with returned message ID {} from producer {}", producer.getTopic(), msgId, getRemote().getInetSocketAddress().toString());
        }
        updateSentMsgStats(msgSize, TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - now));
        if (isConnected()) {
            String messageId = Base64.getEncoder().encodeToString(msgId.toByteArray());
            sendAckResponse(new ProducerAck(messageId, sendRequest.context));
        }
    }).exceptionally(exception -> {
        log.warn("[{}] Error occurred while producer handler was sending msg from {}: {}", producer.getTopic(), getRemote().getInetSocketAddress().toString(), exception.getMessage());
        numMsgsFailed.increment();
        sendAckResponse(new ProducerAck(UnknownError, exception.getMessage(), null, sendRequest.context));
        return null;
    });
}

Example 4 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project pulsar by apache.

the class ContextImplTest method setup.

@BeforeMethod
public void setup() throws PulsarClientException {
    config = new InstanceConfig();
    config.setExposePulsarAdminClientEnabled(true);
    FunctionDetails functionDetails = FunctionDetails.newBuilder().setUserConfig("").build();
    config.setFunctionDetails(functionDetails);
    logger = mock(Logger.class);
    pulsarAdmin = mock(PulsarAdmin.class);
    client = mock(PulsarClientImpl.class);
    when(client.newProducer()).thenReturn(new ProducerBuilderImpl(client, Schema.BYTES));
    when(client.createProducerAsync(any(ProducerConfigurationData.class), any(), any())).thenReturn(CompletableFuture.completedFuture(producer));
    when(client.getSchema(anyString())).thenReturn(CompletableFuture.completedFuture(Optional.empty()));
    when(producer.sendAsync(anyString())).thenReturn(CompletableFuture.completedFuture(null));
    clientBuilder = mock(ClientBuilder.class);
    when(clientBuilder.build()).thenReturn(client);
    TypedMessageBuilder messageBuilder = spy(new TypedMessageBuilderImpl(mock(ProducerBase.class), Schema.STRING));
    doReturn(new CompletableFuture<>()).when(messageBuilder).sendAsync();
    when(producer.newMessage()).thenReturn(messageBuilder);
    context = new ContextImpl(config, logger, client, new EnvironmentBasedSecretsProvider(), FunctionCollectorRegistry.getDefaultImplementation(), new String[0], FunctionDetails.ComponentType.FUNCTION, null, new InstanceStateManager(), pulsarAdmin, clientBuilder);
    context.setCurrentMessageContext((Record<String>) () -> null);
}

Example 5 with TypedMessageBuilder

use of org.apache.pulsar.client.api.TypedMessageBuilder in project pulsar by apache.

the class PulsarSinkTest method getPulsarClient.

/**
 * Verify that JavaInstance does not support functions that take Void type as input.
 */
private static PulsarClientImpl getPulsarClient() throws PulsarClientException {
    PulsarClientImpl pulsarClient = mock(PulsarClientImpl.class);
    ConsumerBuilder consumerBuilder = mock(ConsumerBuilder.class);
    doReturn(consumerBuilder).when(consumerBuilder).topics(anyList());
    doReturn(consumerBuilder).when(consumerBuilder).subscriptionName(anyString());
    doReturn(consumerBuilder).when(consumerBuilder).subscriptionType(any());
    doReturn(consumerBuilder).when(consumerBuilder).ackTimeout(anyLong(), any());
    Consumer consumer = mock(Consumer.class);
    doReturn(consumer).when(consumerBuilder).subscribe();
    doReturn(consumerBuilder).when(pulsarClient).newConsumer(any());
    doReturn(CompletableFuture.completedFuture(Optional.empty())).when(pulsarClient).getSchema(anyString());
    ProducerBuilder producerBuilder = mock(ProducerBuilder.class);
    doReturn(producerBuilder).when(producerBuilder).blockIfQueueFull(anyBoolean());
    doReturn(producerBuilder).when(producerBuilder).enableBatching(anyBoolean());
    doReturn(producerBuilder).when(producerBuilder).batchingMaxPublishDelay(anyLong(), any());
    doReturn(producerBuilder).when(producerBuilder).compressionType(any());
    doReturn(producerBuilder).when(producerBuilder).hashingScheme(any());
    doReturn(producerBuilder).when(producerBuilder).messageRoutingMode(any());
    doReturn(producerBuilder).when(producerBuilder).messageRouter(any());
    doReturn(producerBuilder).when(producerBuilder).topic(anyString());
    doReturn(producerBuilder).when(producerBuilder).producerName(anyString());
    doReturn(producerBuilder).when(producerBuilder).property(anyString(), anyString());
    doReturn(producerBuilder).when(producerBuilder).properties(any());
    doReturn(producerBuilder).when(producerBuilder).sendTimeout(anyInt(), any());
    CompletableFuture completableFuture = new CompletableFuture<>();
    completableFuture.complete(mock(MessageId.class));
    TypedMessageBuilder typedMessageBuilder = mock(TypedMessageBuilder.class);
    doReturn(completableFuture).when(typedMessageBuilder).sendAsync();
    Producer producer = mock(Producer.class);
    doReturn(producer).when(producerBuilder).create();
    doReturn(typedMessageBuilder).when(producer).newMessage();
    doReturn(typedMessageBuilder).when(producer).newMessage(any(Schema.class));
    doReturn(producerBuilder).when(pulsarClient).newProducer();
    doReturn(producerBuilder).when(pulsarClient).newProducer(any());
    return pulsarClient;
}