Examples with PulsarSink org.apache.pulsar.functions.sink.PulsarSink used on opensource projects

Example 1 with PulsarSink

use of org.apache.pulsar.functions.sink.PulsarSink in project pulsar by yahoo.

the class JavaInstanceRunnable method setupOutput.

private void setupOutput(ContextImpl contextImpl) throws Exception {
    SinkSpec sinkSpec = this.instanceConfig.getFunctionDetails().getSink();
    Object object;
    // If sink classname is not set, we default pulsar sink
    if (sinkSpec.getClassName().isEmpty()) {
        if (StringUtils.isEmpty(sinkSpec.getTopic())) {
            object = PulsarSinkDisable.INSTANCE;
        } else {
            PulsarSinkConfig pulsarSinkConfig = new PulsarSinkConfig();
            pulsarSinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.valueOf(this.instanceConfig.getFunctionDetails().getProcessingGuarantees().name()));
            pulsarSinkConfig.setTopic(sinkSpec.getTopic());
            pulsarSinkConfig.setForwardSourceMessageProperty(this.instanceConfig.getFunctionDetails().getSink().getForwardSourceMessageProperty());
            if (!StringUtils.isEmpty(sinkSpec.getSchemaType())) {
                pulsarSinkConfig.setSchemaType(sinkSpec.getSchemaType());
            } else if (!StringUtils.isEmpty(sinkSpec.getSerDeClassName())) {
                pulsarSinkConfig.setSerdeClassName(sinkSpec.getSerDeClassName());
            }
            pulsarSinkConfig.setTypeClassName(sinkSpec.getTypeClassName());
            pulsarSinkConfig.setSchemaProperties(sinkSpec.getSchemaPropertiesMap());
            if (this.instanceConfig.getFunctionDetails().getSink().getProducerSpec() != null) {
                org.apache.pulsar.functions.proto.Function.ProducerSpec conf = this.instanceConfig.getFunctionDetails().getSink().getProducerSpec();
                ProducerConfig.ProducerConfigBuilder builder = ProducerConfig.builder().maxPendingMessages(conf.getMaxPendingMessages()).maxPendingMessagesAcrossPartitions(conf.getMaxPendingMessagesAcrossPartitions()).batchBuilder(conf.getBatchBuilder()).useThreadLocalProducers(conf.getUseThreadLocalProducers()).cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
                pulsarSinkConfig.setProducerConfig(builder.build());
            }
            object = new PulsarSink(this.client, pulsarSinkConfig, this.properties, this.stats, this.functionClassLoader);
        }
    } else {
        object = Reflections.createInstance(sinkSpec.getClassName(), this.functionClassLoader);
    }
    if (object instanceof Sink) {
        this.sink = (Sink) object;
    } else {
        throw new RuntimeException("Sink does not implement correct interface");
    }
    if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) {
        Thread.currentThread().setContextClassLoader(this.functionClassLoader);
    }
    try {
        if (sinkSpec.getConfigs().isEmpty()) {
            if (log.isDebugEnabled()) {
                log.debug("Opening Sink with empty hashmap with contextImpl: {} ", contextImpl.toString());
            }
            this.sink.open(new HashMap<>(), contextImpl);
        } else {
            if (log.isDebugEnabled()) {
                log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec, contextImpl.toString());
            }
            this.sink.open(ObjectMapperFactory.getThreadLocal().readValue(sinkSpec.getConfigs(), new TypeReference<Map<String, Object>>() {
            }), contextImpl);
        }
    } catch (Exception e) {
        log.error("Sink open produced uncaught exception: ", e);
        throw e;
    } finally {
        Thread.currentThread().setContextClassLoader(this.instanceClassLoader);
    }
}

Example 2 with PulsarSink

use of org.apache.pulsar.functions.sink.PulsarSink in project incubator-pulsar by apache.

the class JavaInstanceRunnable method setupOutput.

private void setupOutput(ContextImpl contextImpl) throws Exception {
    SinkSpec sinkSpec = this.instanceConfig.getFunctionDetails().getSink();
    Object object;
    // If sink classname is not set, we default pulsar sink
    if (sinkSpec.getClassName().isEmpty()) {
        if (StringUtils.isEmpty(sinkSpec.getTopic())) {
            object = PulsarSinkDisable.INSTANCE;
        } else {
            PulsarSinkConfig pulsarSinkConfig = new PulsarSinkConfig();
            pulsarSinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.valueOf(this.instanceConfig.getFunctionDetails().getProcessingGuarantees().name()));
            pulsarSinkConfig.setTopic(sinkSpec.getTopic());
            pulsarSinkConfig.setForwardSourceMessageProperty(this.instanceConfig.getFunctionDetails().getSink().getForwardSourceMessageProperty());
            if (!StringUtils.isEmpty(sinkSpec.getSchemaType())) {
                pulsarSinkConfig.setSchemaType(sinkSpec.getSchemaType());
            } else if (!StringUtils.isEmpty(sinkSpec.getSerDeClassName())) {
                pulsarSinkConfig.setSerdeClassName(sinkSpec.getSerDeClassName());
            }
            pulsarSinkConfig.setTypeClassName(sinkSpec.getTypeClassName());
            pulsarSinkConfig.setSchemaProperties(sinkSpec.getSchemaPropertiesMap());
            if (this.instanceConfig.getFunctionDetails().getSink().getProducerSpec() != null) {
                org.apache.pulsar.functions.proto.Function.ProducerSpec conf = this.instanceConfig.getFunctionDetails().getSink().getProducerSpec();
                ProducerConfig.ProducerConfigBuilder builder = ProducerConfig.builder().maxPendingMessages(conf.getMaxPendingMessages()).maxPendingMessagesAcrossPartitions(conf.getMaxPendingMessagesAcrossPartitions()).batchBuilder(conf.getBatchBuilder()).useThreadLocalProducers(conf.getUseThreadLocalProducers()).cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
                pulsarSinkConfig.setProducerConfig(builder.build());
            }
            object = new PulsarSink(this.client, pulsarSinkConfig, this.properties, this.stats, this.functionClassLoader);
        }
    } else {
        object = Reflections.createInstance(sinkSpec.getClassName(), this.functionClassLoader);
    }
    if (object instanceof Sink) {
        this.sink = (Sink) object;
    } else {
        throw new RuntimeException("Sink does not implement correct interface");
    }
    if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) {
        Thread.currentThread().setContextClassLoader(this.functionClassLoader);
    }
    try {
        if (sinkSpec.getConfigs().isEmpty()) {
            if (log.isDebugEnabled()) {
                log.debug("Opening Sink with empty hashmap with contextImpl: {} ", contextImpl.toString());
            }
            this.sink.open(new HashMap<>(), contextImpl);
        } else {
            if (log.isDebugEnabled()) {
                log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec, contextImpl.toString());
            }
            this.sink.open(ObjectMapperFactory.getThreadLocal().readValue(sinkSpec.getConfigs(), new TypeReference<Map<String, Object>>() {
            }), contextImpl);
        }
    } catch (Exception e) {
        log.error("Sink open produced uncaught exception: ", e);
        throw e;
    } finally {
        Thread.currentThread().setContextClassLoader(this.instanceClassLoader);
    }
}

Example 3 with PulsarSink

use of org.apache.pulsar.functions.sink.PulsarSink in project pulsar by apache.

the class JavaInstanceRunnable method setupOutput.

private void setupOutput(ContextImpl contextImpl) throws Exception {
    SinkSpec sinkSpec = this.instanceConfig.getFunctionDetails().getSink();
    Object object;
    // If sink classname is not set, we default pulsar sink
    if (sinkSpec.getClassName().isEmpty()) {
        if (StringUtils.isEmpty(sinkSpec.getTopic())) {
            object = PulsarSinkDisable.INSTANCE;
        } else {
            PulsarSinkConfig pulsarSinkConfig = new PulsarSinkConfig();
            pulsarSinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.valueOf(this.instanceConfig.getFunctionDetails().getProcessingGuarantees().name()));
            pulsarSinkConfig.setTopic(sinkSpec.getTopic());
            pulsarSinkConfig.setForwardSourceMessageProperty(this.instanceConfig.getFunctionDetails().getSink().getForwardSourceMessageProperty());
            if (!StringUtils.isEmpty(sinkSpec.getSchemaType())) {
                pulsarSinkConfig.setSchemaType(sinkSpec.getSchemaType());
            } else if (!StringUtils.isEmpty(sinkSpec.getSerDeClassName())) {
                pulsarSinkConfig.setSerdeClassName(sinkSpec.getSerDeClassName());
            }
            pulsarSinkConfig.setTypeClassName(sinkSpec.getTypeClassName());
            pulsarSinkConfig.setSchemaProperties(sinkSpec.getSchemaPropertiesMap());
            if (this.instanceConfig.getFunctionDetails().getSink().getProducerSpec() != null) {
                org.apache.pulsar.functions.proto.Function.ProducerSpec conf = this.instanceConfig.getFunctionDetails().getSink().getProducerSpec();
                ProducerConfig.ProducerConfigBuilder builder = ProducerConfig.builder().maxPendingMessages(conf.getMaxPendingMessages()).maxPendingMessagesAcrossPartitions(conf.getMaxPendingMessagesAcrossPartitions()).batchBuilder(conf.getBatchBuilder()).useThreadLocalProducers(conf.getUseThreadLocalProducers()).cryptoConfig(CryptoUtils.convertFromSpec(conf.getCryptoSpec()));
                pulsarSinkConfig.setProducerConfig(builder.build());
            }
            object = new PulsarSink(this.client, pulsarSinkConfig, this.properties, this.stats, this.functionClassLoader);
        }
    } else {
        object = Reflections.createInstance(sinkSpec.getClassName(), this.functionClassLoader);
    }
    if (object instanceof Sink) {
        this.sink = (Sink) object;
    } else {
        throw new RuntimeException("Sink does not implement correct interface");
    }
    if (componentType == org.apache.pulsar.functions.proto.Function.FunctionDetails.ComponentType.SINK) {
        Thread.currentThread().setContextClassLoader(this.functionClassLoader);
    }
    try {
        if (sinkSpec.getConfigs().isEmpty()) {
            if (log.isDebugEnabled()) {
                log.debug("Opening Sink with empty hashmap with contextImpl: {} ", contextImpl.toString());
            }
            this.sink.open(new HashMap<>(), contextImpl);
        } else {
            if (log.isDebugEnabled()) {
                log.debug("Opening Sink with SinkSpec {} and contextImpl: {} ", sinkSpec.toString(), contextImpl.toString());
            }
            this.sink.open(ObjectMapperFactory.getThreadLocal().readValue(sinkSpec.getConfigs(), new TypeReference<Map<String, Object>>() {
            }), contextImpl);
        }
    } catch (Exception e) {
        log.error("Sink open produced uncaught exception: ", e);
        throw e;
    } finally {
        Thread.currentThread().setContextClassLoader(this.instanceClassLoader);
    }
}