Examples with FunctionConfig org.apache.pulsar.functions.proto.Function.FunctionConfig used on opensource projects

Example 11 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testEffectivelyOnceProcessing.

@Test
public void testEffectivelyOnceProcessing() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.EFFECTIVELY_ONCE).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        ProducerInstance producerInstance;
        while (mockProducers.isEmpty()) {
            TimeUnit.MILLISECONDS.sleep(20);
        }
        producerInstance = mockProducers.values().iterator().next();
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(Utils.getSequenceId(new MessageIdImpl(1L, i, 0)), msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeCumulativeAsync(any(Message.class));
        assertEquals(10, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // acknowledges count should remain same
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeCumulativeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
    }
}

Example 12 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtMostOnceProcessing.

@Test
public void testAtMostOnceProcessing() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // acknowledges count should remain same
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
    }
}

Example 13 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class FunctionConfigTest method testDefaultProcessingGuarantee.

/**
 * Make sure the default processing guarantee is always `ATLEAST_ONCE`.
 */
@Test
public void testDefaultProcessingGuarantee() {
    FunctionConfig fc = FunctionConfig.newBuilder().build();
    assertEquals(ProcessingGuarantees.ATLEAST_ONCE, fc.getProcessingGuarantees());
}

Example 14 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class JavaInstanceMain method start.

public void start() throws Exception {
    InstanceConfig instanceConfig = new InstanceConfig();
    instanceConfig.setFunctionId(functionId);
    instanceConfig.setFunctionVersion(functionVersion);
    instanceConfig.setInstanceId(instanceId);
    instanceConfig.setMaxBufferedTuples(maxBufferedTuples);
    FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
    functionConfigBuilder.setTenant(tenant);
    functionConfigBuilder.setNamespace(namespace);
    functionConfigBuilder.setName(functionName);
    functionConfigBuilder.setClassName(className);
    if (defaultSerdeInputTopics != null) {
        String[] inputTopics = defaultSerdeInputTopics.split(",");
        for (String inputTopic : inputTopics) {
            functionConfigBuilder.addInputs(inputTopic);
        }
    }
    if (customSerdeInputTopics != null && customSerdeClassnames != null) {
        String[] inputTopics = customSerdeInputTopics.split(",");
        String[] inputSerdeClassNames = customSerdeClassnames.split(",");
        if (inputTopics.length != inputSerdeClassNames.length) {
            throw new RuntimeException("Error specifying inputs");
        }
        for (int i = 0; i < inputTopics.length; ++i) {
            functionConfigBuilder.putCustomSerdeInputs(inputTopics[i], inputSerdeClassNames[i]);
        }
    }
    if (outputSerdeClassName != null) {
        functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName);
    }
    if (outputTopicName != null) {
        functionConfigBuilder.setOutput(outputTopicName);
    }
    if (logTopic != null) {
        functionConfigBuilder.setLogTopic(logTopic);
    }
    functionConfigBuilder.setProcessingGuarantees(processingGuarantees);
    if (autoAck.equals("true")) {
        functionConfigBuilder.setAutoAck(true);
    } else {
        functionConfigBuilder.setAutoAck(false);
    }
    if (userConfig != null && !userConfig.isEmpty()) {
        Type type = new TypeToken<Map<String, String>>() {
        }.getType();
        Map<String, String> userConfigMap = new Gson().fromJson(userConfig, type);
        functionConfigBuilder.putAllUserConfig(userConfigMap);
    }
    FunctionConfig functionConfig = functionConfigBuilder.build();
    instanceConfig.setFunctionConfig(functionConfig);
    ThreadRuntimeFactory containerFactory = new ThreadRuntimeFactory("LocalRunnerThreadGroup", pulsarServiceUrl, stateStorageServiceUrl);
    RuntimeSpawner runtimeSpawner = new RuntimeSpawner(instanceConfig, jarFile, containerFactory, null);
    server = ServerBuilder.forPort(port).addService(new InstanceControlImpl(runtimeSpawner)).build().start();
    log.info("JaveInstance Server started, listening on " + port);
    java.lang.Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            // Use stderr here since the logger may have been reset by its JVM shutdown hook.
            try {
                server.shutdown();
                runtimeSpawner.close();
            } catch (Exception ex) {
                System.err.println(ex);
            }
        }
    });
    log.info("Starting runtimeSpawner");
    runtimeSpawner.start();
    runtimeSpawner.join();
    log.info("RuntimeSpawner quit, shutting down JavaInstance");
    server.shutdown();
}

Example 15 with FunctionConfig

use of org.apache.pulsar.functions.proto.Function.FunctionConfig in project incubator-pulsar by apache.

the class FunctionApiV2ResourceTest method testUpdateFunctionMissingArguments.

private void testUpdateFunctionMissingArguments(String tenant, String namespace, String function, InputStream inputStream, FormDataContentDisposition details, String outputTopic, String inputTopic, String inputSerdeClassName, String outputSerdeClassName, String className, Integer parallelism, String missingFieldName) throws IOException {
    FunctionConfig.Builder functionConfigBuilder = FunctionConfig.newBuilder();
    if (tenant != null) {
        functionConfigBuilder.setTenant(tenant);
    }
    if (namespace != null) {
        functionConfigBuilder.setNamespace(namespace);
    }
    if (function != null) {
        functionConfigBuilder.setName(function);
    }
    if (outputTopic != null) {
        functionConfigBuilder.setOutput(outputTopic);
    }
    if (inputTopic != null && inputSerdeClassName != null) {
        functionConfigBuilder.putCustomSerdeInputs(inputTopic, inputSerdeClassName);
    }
    if (outputSerdeClassName != null) {
        functionConfigBuilder.setOutputSerdeClassName(outputSerdeClassName);
    }
    if (className != null) {
        functionConfigBuilder.setClassName(className);
    }
    if (parallelism != null) {
        functionConfigBuilder.setParallelism(parallelism);
    }
    FunctionConfig functionConfig = functionConfigBuilder.build();
    Response response = resource.updateFunction(tenant, namespace, function, inputStream, details, org.apache.pulsar.functions.utils.Utils.printJson(functionConfig));
    assertEquals(Status.BAD_REQUEST.getStatusCode(), response.getStatus());
    if (missingFieldName.equals("parallelism")) {
        Assert.assertEquals(new ErrorData("Parallelism needs to be set to a positive number").reason, ((ErrorData) response.getEntity()).reason);
    } else {
        Assert.assertEquals(new ErrorData(missingFieldName + " is not provided").reason, ((ErrorData) response.getEntity()).reason);
    }
}