Examples with SerDe org.apache.pulsar.functions.api.SerDe used on opensource projects

Example 1 with SerDe

use of org.apache.pulsar.functions.api.SerDe in project incubator-pulsar by apache.

the class DefaultSerDeTest method testPulsarFunction.

@Test
public void testPulsarFunction() {
    SimplePulsarFunction pulsarFunction = new SimplePulsarFunction();
    Class<?>[] typeArgs = TypeResolver.resolveRawArguments(Function.class, pulsarFunction.getClass());
    SerDe serDe = new DefaultSerDe(String.class);
    Class<?>[] inputSerdeTypeArgs = TypeResolver.resolveRawArguments(SerDe.class, serDe.getClass());
    assertTrue(inputSerdeTypeArgs[0].isAssignableFrom(typeArgs[0]));
}

Example 2 with SerDe

use of org.apache.pulsar.functions.api.SerDe in project incubator-pulsar by apache.

the class JavaInstanceRunnable method createConsumerConfiguration.

private ConsumerConfiguration createConsumerConfiguration(String topicName) {
    log.info("Starting Consumer for topic " + topicName);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(getSubscriptionType());
    SerDe inputSerde = inputSerDe.get(topicName);
    conf.setMessageListener((consumer, msg) -> {
        try {
            InputMessage message = new InputMessage();
            message.setConsumer(consumer);
            message.setInputSerDe(inputSerde);
            message.setActualMessage(msg);
            message.setTopicName(topicName);
            queue.put(message);
            if (processingGuarantees == FunctionConfig.ProcessingGuarantees.ATMOST_ONCE) {
                if (instanceConfig.getFunctionConfig().getAutoAck()) {
                    consumer.acknowledgeAsync(msg);
                }
            }
        } catch (InterruptedException e) {
            log.error("Function container {} is interrupted on enqueuing messages", Thread.currentThread().getId(), e);
        }
    });
    // for failover subscription, register a consumer event listener to react to active consumer changes.
    if (getSubscriptionType() == SubscriptionType.Failover) {
        conf.setConsumerEventListener(this);
    }
    return conf;
}

Example 3 with SerDe

use of org.apache.pulsar.functions.api.SerDe in project incubator-pulsar by apache.

the class JavaInstanceRunnable method setupSerDe.

private void setupSerDe(Class<?>[] typeArgs, ClassLoader clsLoader) {
    this.inputSerDe = new HashMap<>();
    instanceConfig.getFunctionConfig().getCustomSerdeInputsMap().forEach((k, v) -> this.inputSerDe.put(k, initializeSerDe(v, clsLoader, typeArgs, true)));
    for (String topicName : instanceConfig.getFunctionConfig().getInputsList()) {
        this.inputSerDe.put(topicName, initializeDefaultSerDe(typeArgs, true));
    }
    if (Void.class.equals(typeArgs[0])) {
        throw new RuntimeException("Input type of Pulsar Function cannot be Void");
    }
    for (SerDe serDe : inputSerDe.values()) {
        if (serDe.getClass().getName().equals(DefaultSerDe.class.getName())) {
            if (!DefaultSerDe.IsSupportedType(typeArgs[0])) {
                throw new RuntimeException("Default Serde does not support " + typeArgs[0]);
            }
        } else {
            Class<?>[] inputSerdeTypeArgs = TypeResolver.resolveRawArguments(SerDe.class, serDe.getClass());
            if (!typeArgs[0].isAssignableFrom(inputSerdeTypeArgs[0])) {
                throw new RuntimeException("Inconsistent types found between function input type and input serde type: " + " function type = " + typeArgs[0] + " should be assignable from " + inputSerdeTypeArgs[0]);
            }
        }
    }
    if (!Void.class.equals(typeArgs[1])) {
        // return type is not `Void.class`
        if (instanceConfig.getFunctionConfig().getOutputSerdeClassName() == null || instanceConfig.getFunctionConfig().getOutputSerdeClassName().isEmpty() || instanceConfig.getFunctionConfig().getOutputSerdeClassName().equals(DefaultSerDe.class.getName())) {
            outputSerDe = initializeDefaultSerDe(typeArgs, false);
        } else {
            this.outputSerDe = initializeSerDe(instanceConfig.getFunctionConfig().getOutputSerdeClassName(), clsLoader, typeArgs, false);
        }
        Class<?>[] outputSerdeTypeArgs = TypeResolver.resolveRawArguments(SerDe.class, outputSerDe.getClass());
        if (outputSerDe.getClass().getName().equals(DefaultSerDe.class.getName())) {
            if (!DefaultSerDe.IsSupportedType(typeArgs[1])) {
                throw new RuntimeException("Default Serde does not support type " + typeArgs[1]);
            }
        } else if (!outputSerdeTypeArgs[0].isAssignableFrom(typeArgs[1])) {
            throw new RuntimeException("Inconsistent types found between function output type and output serde type: " + " function type = " + typeArgs[1] + "should be assignable from " + outputSerdeTypeArgs[0]);
        }
    }
}

Example 4 with SerDe

use of org.apache.pulsar.functions.api.SerDe in project incubator-pulsar by apache.

the class ContextImpl method publish.

@Override
public <O> CompletableFuture<Void> publish(String topicName, O object, String serDeClassName) {
    if (!publishProducers.containsKey(topicName)) {
        try {
            publishProducers.put(topicName, pulsarClient.createProducer(topicName, producerConfiguration));
        } catch (PulsarClientException ex) {
            CompletableFuture<Void> retval = new CompletableFuture<>();
            retval.completeExceptionally(ex);
            return retval;
        }
    }
    if (!publishSerializers.containsKey(serDeClassName)) {
        SerDe serDe;
        if (serDeClassName.equals(DefaultSerDe.class.getName())) {
            if (!DefaultSerDe.IsSupportedType(object.getClass())) {
                throw new RuntimeException("Default Serializer does not support " + object.getClass());
            }
            serDe = new DefaultSerDe(object.getClass());
        } else {
            try {
                Class<? extends SerDe> serDeClass = (Class<? extends SerDe>) Class.forName(serDeClassName);
                serDe = Reflections.createInstance(serDeClassName, serDeClass, classLoader);
            } catch (ClassNotFoundException e) {
                throw new RuntimeException(e);
            }
        }
        publishSerializers.put(serDeClassName, serDe);
    }
    byte[] bytes = publishSerializers.get(serDeClassName).serialize(object);
    return publishProducers.get(topicName).sendAsync(bytes).thenApply(msgId -> null);
}