Examples with FunctionDetails org.apache.pulsar.functions.proto.Function.FunctionDetails used on opensource projects

Example 1 with FunctionDetails

use of org.apache.pulsar.functions.proto.Function.FunctionDetails in project pulsar by apache.

the class FunctionActioner method terminateFunction.

public void terminateFunction(FunctionRuntimeInfo functionRuntimeInfo) {
    FunctionDetails details = functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().getFunctionDetails();
    String fqfn = FunctionCommon.getFullyQualifiedName(details);
    log.info("{}-{} Terminating function...", fqfn, functionRuntimeInfo.getFunctionInstance().getInstanceId());
    if (functionRuntimeInfo.getRuntimeSpawner() != null) {
        functionRuntimeInfo.getRuntimeSpawner().close();
        // cleanup any auth data cached
        if (workerConfig.isAuthenticationEnabled()) {
            functionRuntimeInfo.getRuntimeSpawner().getRuntimeFactory().getAuthProvider().ifPresent(functionAuthProvider -> {
                try {
                    log.info("{}-{} Cleaning up authentication data for function...", fqfn, functionRuntimeInfo.getFunctionInstance().getInstanceId());
                    functionAuthProvider.cleanUpAuthData(details, Optional.ofNullable(getFunctionAuthData(Optional.ofNullable(functionRuntimeInfo.getRuntimeSpawner().getInstanceConfig().getFunctionAuthenticationSpec()))));
                } catch (Exception e) {
                    log.error("Failed to cleanup auth data for function: {}", fqfn, e);
                }
            });
        }
        functionRuntimeInfo.setRuntimeSpawner(null);
    }
    cleanupFunctionFiles(functionRuntimeInfo);
    // cleanup subscriptions
    if (details.getSource().getCleanupSubscription()) {
        Map<String, Function.ConsumerSpec> consumerSpecMap = details.getSource().getInputSpecsMap();
        consumerSpecMap.entrySet().forEach(new Consumer<Map.Entry<String, Function.ConsumerSpec>>() {

            @Override
            public void accept(Map.Entry<String, Function.ConsumerSpec> stringConsumerSpecEntry) {
                Function.ConsumerSpec consumerSpec = stringConsumerSpecEntry.getValue();
                String topic = stringConsumerSpecEntry.getKey();
                String subscriptionName = isBlank(functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().getFunctionDetails().getSource().getSubscriptionName()) ? InstanceUtils.getDefaultSubscriptionName(functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().getFunctionDetails()) : functionRuntimeInfo.getFunctionInstance().getFunctionMetaData().getFunctionDetails().getSource().getSubscriptionName();
                deleteSubscription(topic, consumerSpec, subscriptionName, String.format("Cleaning up subscriptions for function %s", fqfn));
            }
        });
    }
    // clean up done for batch sources if necessary
    cleanupBatchSource(details);
}

Example 2 with FunctionDetails

use of org.apache.pulsar.functions.proto.Function.FunctionDetails in project pulsar by apache.

the class FunctionActioner method stopFunction.

public void stopFunction(FunctionRuntimeInfo functionRuntimeInfo) {
    Function.Instance instance = functionRuntimeInfo.getFunctionInstance();
    FunctionMetaData functionMetaData = instance.getFunctionMetaData();
    FunctionDetails details = functionMetaData.getFunctionDetails();
    log.info("{}/{}/{}-{} Stopping function...", details.getTenant(), details.getNamespace(), details.getName(), instance.getInstanceId());
    if (functionRuntimeInfo.getRuntimeSpawner() != null) {
        functionRuntimeInfo.getRuntimeSpawner().close();
        functionRuntimeInfo.setRuntimeSpawner(null);
    }
    cleanupFunctionFiles(functionRuntimeInfo);
}

Example 3 with FunctionDetails

use of org.apache.pulsar.functions.proto.Function.FunctionDetails in project pulsar by apache.

the class FunctionConfigUtilsTest method testFunctionConfigConvertFromDetails.

@Test
public void testFunctionConfigConvertFromDetails() {
    String name = "test1";
    String namespace = "ns1";
    String tenant = "tenant1";
    String classname = getClass().getName();
    int parallelism = 3;
    Map<String, String> userConfig = new HashMap<>();
    userConfig.put("key1", "val1");
    Function.ProcessingGuarantees processingGuarantees = Function.ProcessingGuarantees.EFFECTIVELY_ONCE;
    Function.FunctionDetails.Runtime runtime = Function.FunctionDetails.Runtime.JAVA;
    Function.SinkSpec sinkSpec = Function.SinkSpec.newBuilder().setTopic("sinkTopic1").build();
    Map<String, Function.ConsumerSpec> consumerSpecMap = new HashMap<>();
    consumerSpecMap.put("sourceTopic1", Function.ConsumerSpec.newBuilder().setSchemaType(JSONSchema.class.getName()).build());
    Function.SourceSpec sourceSpec = Function.SourceSpec.newBuilder().putAllInputSpecs(consumerSpecMap).setSubscriptionType(Function.SubscriptionType.FAILOVER).setCleanupSubscription(true).build();
    boolean autoAck = true;
    String logTopic = "log-topic1";
    Function.Resources resources = Function.Resources.newBuilder().setCpu(1.5).setDisk(1024 * 20).setRam(1024 * 10).build();
    String packageUrl = "http://package.url";
    Map<String, String> secretsMap = new HashMap<>();
    secretsMap.put("secretConfigKey1", "secretConfigVal1");
    Function.RetryDetails retryDetails = Function.RetryDetails.newBuilder().setDeadLetterTopic("dead-letter-1").build();
    Function.FunctionDetails functionDetails = Function.FunctionDetails.newBuilder().setNamespace(namespace).setTenant(tenant).setName(name).setClassName(classname).setParallelism(parallelism).setUserConfig(new Gson().toJson(userConfig)).setProcessingGuarantees(processingGuarantees).setRuntime(runtime).setSink(sinkSpec).setSource(sourceSpec).setAutoAck(autoAck).setLogTopic(logTopic).setResources(resources).setPackageUrl(packageUrl).setSecretsMap(new Gson().toJson(secretsMap)).setRetryDetails(retryDetails).build();
    FunctionConfig functionConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
    assertEquals(functionConfig.getTenant(), tenant);
    assertEquals(functionConfig.getNamespace(), namespace);
    assertEquals(functionConfig.getName(), name);
    assertEquals(functionConfig.getClassName(), classname);
    assertEquals(functionConfig.getLogTopic(), logTopic);
    assertEquals((Object) functionConfig.getResources().getCpu(), resources.getCpu());
    assertEquals(functionConfig.getResources().getDisk().longValue(), resources.getDisk());
    assertEquals(functionConfig.getResources().getRam().longValue(), resources.getRam());
    assertEquals(functionConfig.getOutput(), sinkSpec.getTopic());
    assertEquals(functionConfig.getInputSpecs().keySet(), sourceSpec.getInputSpecsMap().keySet());
    assertEquals(functionConfig.getCleanupSubscription().booleanValue(), sourceSpec.getCleanupSubscription());
}

Example 4 with FunctionDetails

use of org.apache.pulsar.functions.proto.Function.FunctionDetails in project pulsar by apache.

the class FunctionConfigUtilsTest method testConvertBackFidelity.

@Test
public void testConvertBackFidelity() {
    FunctionConfig functionConfig = new FunctionConfig();
    functionConfig.setTenant("test-tenant");
    functionConfig.setNamespace("test-namespace");
    functionConfig.setName("test-function");
    functionConfig.setParallelism(1);
    functionConfig.setClassName(IdentityFunction.class.getName());
    Map<String, ConsumerConfig> inputSpecs = new HashMap<>();
    inputSpecs.put("test-input", ConsumerConfig.builder().isRegexPattern(true).serdeClassName("test-serde").poolMessages(true).build());
    functionConfig.setInputSpecs(inputSpecs);
    functionConfig.setOutput("test-output");
    functionConfig.setOutputSerdeClassName("test-serde");
    functionConfig.setRuntime(FunctionConfig.Runtime.JAVA);
    functionConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    functionConfig.setRetainOrdering(false);
    functionConfig.setRetainKeyOrdering(false);
    functionConfig.setForwardSourceMessageProperty(true);
    functionConfig.setUserConfig(new HashMap<>());
    functionConfig.setAutoAck(true);
    functionConfig.setTimeoutMs(2000l);
    functionConfig.setRuntimeFlags("-DKerberos");
    ProducerConfig producerConfig = new ProducerConfig();
    producerConfig.setMaxPendingMessages(100);
    producerConfig.setMaxPendingMessagesAcrossPartitions(1000);
    producerConfig.setUseThreadLocalProducers(true);
    producerConfig.setBatchBuilder("DEFAULT");
    functionConfig.setProducerConfig(producerConfig);
    Function.FunctionDetails functionDetails = FunctionConfigUtils.convert(functionConfig, null);
    FunctionConfig convertedConfig = FunctionConfigUtils.convertFromDetails(functionDetails);
    // add default resources
    functionConfig.setResources(Resources.getDefaultResources());
    // set default cleanupSubscription config
    functionConfig.setCleanupSubscription(true);
    assertEquals(new Gson().toJson(functionConfig), new Gson().toJson(convertedConfig));
}

Example 5 with FunctionDetails

use of org.apache.pulsar.functions.proto.Function.FunctionDetails in project pulsar by apache.

the class ContextImplTest method setup.

@BeforeMethod
public void setup() throws PulsarClientException {
    config = new InstanceConfig();
    config.setExposePulsarAdminClientEnabled(true);
    FunctionDetails functionDetails = FunctionDetails.newBuilder().setUserConfig("").build();
    config.setFunctionDetails(functionDetails);
    logger = mock(Logger.class);
    pulsarAdmin = mock(PulsarAdmin.class);
    client = mock(PulsarClientImpl.class);
    when(client.newProducer()).thenReturn(new ProducerBuilderImpl(client, Schema.BYTES));
    when(client.createProducerAsync(any(ProducerConfigurationData.class), any(), any())).thenReturn(CompletableFuture.completedFuture(producer));
    when(client.getSchema(anyString())).thenReturn(CompletableFuture.completedFuture(Optional.empty()));
    when(producer.sendAsync(anyString())).thenReturn(CompletableFuture.completedFuture(null));
    clientBuilder = mock(ClientBuilder.class);
    when(clientBuilder.build()).thenReturn(client);
    TypedMessageBuilder messageBuilder = spy(new TypedMessageBuilderImpl(mock(ProducerBase.class), Schema.STRING));
    doReturn(new CompletableFuture<>()).when(messageBuilder).sendAsync();
    when(producer.newMessage()).thenReturn(messageBuilder);
    context = new ContextImpl(config, logger, client, new EnvironmentBasedSecretsProvider(), FunctionCollectorRegistry.getDefaultImplementation(), new String[0], FunctionDetails.ComponentType.FUNCTION, null, new InstanceStateManager(), pulsarAdmin, clientBuilder);
    context.setCurrentMessageContext((Record<String>) () -> null);
}