Examples with SinkConfig org.apache.pulsar.common.io.SinkConfig used on opensource projects

Example 26 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class SinkConfigUtilsTest method testMergeDifferentTenant.

@Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "Tenants differ")
public void testMergeDifferentTenant() {
    SinkConfig sinkConfig = createSinkConfig();
    SinkConfig newSinkConfig = createUpdatedSinkConfig("tenant", "Different");
    SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig);
}

Example 27 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class SinkConfigUtilsTest method testMergeDifferentTimeout.

@Test
public void testMergeDifferentTimeout() {
    SinkConfig sinkConfig = createSinkConfig();
    SinkConfig newSinkConfig = createUpdatedSinkConfig("timeoutMs", 102l);
    SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig);
    assertEquals(mergedConfig.getTimeoutMs(), new Long(102l));
    mergedConfig.setTimeoutMs(sinkConfig.getTimeoutMs());
    assertEquals(new Gson().toJson(sinkConfig), new Gson().toJson(mergedConfig));
}

Example 28 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class SinkConfigUtilsTest method testMergeRuntimeFlags.

@Test
public void testMergeRuntimeFlags() {
    SinkConfig sinkConfig = createSinkConfig();
    SinkConfig newFunctionConfig = createUpdatedSinkConfig("runtimeFlags", "-Dfoo=bar2");
    SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newFunctionConfig);
    assertEquals(mergedConfig.getRuntimeFlags(), "-Dfoo=bar2");
    mergedConfig.setRuntimeFlags(sinkConfig.getRuntimeFlags());
    assertEquals(new Gson().toJson(sinkConfig), new Gson().toJson(mergedConfig));
}

Example 29 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class SinkConfigUtilsTest method testMergeDifferentSecrets.

@Test
public void testMergeDifferentSecrets() {
    SinkConfig sinkConfig = createSinkConfig();
    Map<String, String> mySecrets = new HashMap<>();
    mySecrets.put("MyKey", "MyValue");
    SinkConfig newSinkConfig = createUpdatedSinkConfig("secrets", mySecrets);
    SinkConfig mergedConfig = SinkConfigUtils.validateUpdate(sinkConfig, newSinkConfig);
    assertEquals(mergedConfig.getSecrets(), mySecrets);
    mergedConfig.setSecrets(sinkConfig.getSecrets());
    assertEquals(new Gson().toJson(sinkConfig), new Gson().toJson(mergedConfig));
}

Example 30 with SinkConfig

use of org.apache.pulsar.common.io.SinkConfig in project pulsar by apache.

the class SinkConfigUtils method convertFromDetails.

public static SinkConfig convertFromDetails(FunctionDetails functionDetails) {
    SinkConfig sinkConfig = new SinkConfig();
    sinkConfig.setTenant(functionDetails.getTenant());
    sinkConfig.setNamespace(functionDetails.getNamespace());
    sinkConfig.setName(functionDetails.getName());
    sinkConfig.setParallelism(functionDetails.getParallelism());
    sinkConfig.setProcessingGuarantees(FunctionCommon.convertProcessingGuarantee(functionDetails.getProcessingGuarantees()));
    Map<String, ConsumerConfig> consumerConfigMap = new HashMap<>();
    List<String> inputs = new ArrayList<>();
    for (Map.Entry<String, Function.ConsumerSpec> input : functionDetails.getSource().getInputSpecsMap().entrySet()) {
        ConsumerConfig consumerConfig = new ConsumerConfig();
        if (!isEmpty(input.getValue().getSerdeClassName())) {
            consumerConfig.setSerdeClassName(input.getValue().getSerdeClassName());
        }
        if (!isEmpty(input.getValue().getSchemaType())) {
            consumerConfig.setSchemaType(input.getValue().getSchemaType());
        }
        if (input.getValue().hasReceiverQueueSize()) {
            consumerConfig.setReceiverQueueSize(input.getValue().getReceiverQueueSize().getValue());
        }
        if (input.getValue().hasCryptoSpec()) {
            consumerConfig.setCryptoConfig(CryptoUtils.convertFromSpec(input.getValue().getCryptoSpec()));
        }
        consumerConfig.setRegexPattern(input.getValue().getIsRegexPattern());
        consumerConfig.setConsumerProperties(input.getValue().getConsumerPropertiesMap());
        consumerConfig.setPoolMessages(input.getValue().getPoolMessages());
        consumerConfigMap.put(input.getKey(), consumerConfig);
        inputs.add(input.getKey());
    }
    sinkConfig.setInputs(inputs);
    sinkConfig.setInputSpecs(consumerConfigMap);
    if (!isEmpty(functionDetails.getSource().getSubscriptionName())) {
        sinkConfig.setSourceSubscriptionName(functionDetails.getSource().getSubscriptionName());
    }
    if (functionDetails.getSource().getSubscriptionType() == Function.SubscriptionType.FAILOVER) {
        sinkConfig.setRetainOrdering(true);
        sinkConfig.setRetainKeyOrdering(false);
    } else if (functionDetails.getSource().getSubscriptionType() == Function.SubscriptionType.KEY_SHARED) {
        sinkConfig.setRetainOrdering(false);
        sinkConfig.setRetainKeyOrdering(true);
    } else {
        sinkConfig.setRetainOrdering(false);
        sinkConfig.setRetainKeyOrdering(false);
    }
    sinkConfig.setProcessingGuarantees(convertProcessingGuarantee(functionDetails.getProcessingGuarantees()));
    sinkConfig.setAutoAck(functionDetails.getAutoAck());
    sinkConfig.setCleanupSubscription(functionDetails.getSource().getCleanupSubscription());
    // Set subscription position
    sinkConfig.setSourceSubscriptionPosition(convertFromFunctionDetailsSubscriptionPosition(functionDetails.getSource().getSubscriptionPosition()));
    sinkConfig.setCleanupSubscription(functionDetails.getSource().getCleanupSubscription());
    if (functionDetails.getSource().getTimeoutMs() != 0) {
        sinkConfig.setTimeoutMs(functionDetails.getSource().getTimeoutMs());
    }
    if (functionDetails.getSource().getNegativeAckRedeliveryDelayMs() > 0) {
        sinkConfig.setNegativeAckRedeliveryDelayMs(functionDetails.getSource().getNegativeAckRedeliveryDelayMs());
    }
    if (!isEmpty(functionDetails.getSink().getClassName())) {
        sinkConfig.setClassName(functionDetails.getSink().getClassName());
    }
    if (!isEmpty(functionDetails.getSink().getBuiltin())) {
        sinkConfig.setArchive("builtin://" + functionDetails.getSink().getBuiltin());
    }
    if (!org.apache.commons.lang3.StringUtils.isEmpty(functionDetails.getSink().getConfigs())) {
        TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {
        };
        Map<String, Object> configMap;
        try {
            configMap = ObjectMapperFactory.getThreadLocal().readValue(functionDetails.getSink().getConfigs(), typeRef);
        } catch (IOException e) {
            log.error("Failed to read configs for sink {}", FunctionCommon.getFullyQualifiedName(functionDetails), e);
            throw new RuntimeException(e);
        }
        sinkConfig.setConfigs(configMap);
    }
    if (!isEmpty(functionDetails.getSecretsMap())) {
        Type type = new TypeToken<Map<String, Object>>() {
        }.getType();
        Map<String, Object> secretsMap = new Gson().fromJson(functionDetails.getSecretsMap(), type);
        sinkConfig.setSecrets(secretsMap);
    }
    if (functionDetails.hasResources()) {
        Resources resources = new Resources();
        resources.setCpu(functionDetails.getResources().getCpu());
        resources.setRam(functionDetails.getResources().getRam());
        resources.setDisk(functionDetails.getResources().getDisk());
        sinkConfig.setResources(resources);
    }
    if (isNotBlank(functionDetails.getRuntimeFlags())) {
        sinkConfig.setRuntimeFlags(functionDetails.getRuntimeFlags());
    }
    if (!isEmpty(functionDetails.getCustomRuntimeOptions())) {
        sinkConfig.setCustomRuntimeOptions(functionDetails.getCustomRuntimeOptions());
    }
    if (functionDetails.hasRetryDetails()) {
        sinkConfig.setMaxMessageRetries(functionDetails.getRetryDetails().getMaxMessageRetries());
        if (StringUtils.isNotBlank(functionDetails.getRetryDetails().getDeadLetterTopic())) {
            sinkConfig.setDeadLetterTopic(functionDetails.getRetryDetails().getDeadLetterTopic());
        }
    }
    return sinkConfig;
}