Examples with BatchSourceConfig org.apache.pulsar.common.io.BatchSourceConfig used on opensource projects

Example 1 with BatchSourceConfig

use of org.apache.pulsar.common.io.BatchSourceConfig in project pulsar by apache.

the class SourceConfigUtilsTest method testMergeDifferentBatchSourceConfig.

@Test
public void testMergeDifferentBatchSourceConfig() {
    SourceConfig sourceConfig = createSourceConfigWithBatch();
    BatchSourceConfig batchSourceConfig = createBatchSourceConfig();
    Map<String, Object> newConfig = new HashMap<>();
    newConfig.put("something", "different");
    batchSourceConfig.setDiscoveryTriggererConfig(newConfig);
    SourceConfig newSourceConfig = createUpdatedSourceConfig("batchSourceConfig", batchSourceConfig);
    SourceConfig mergedConfig = SourceConfigUtils.validateUpdate(sourceConfig, newSourceConfig);
    assertEquals(mergedConfig.getBatchSourceConfig().getDiscoveryTriggererConfig().get("something"), "different");
    mergedConfig.getBatchSourceConfig().setDiscoveryTriggererConfig(sourceConfig.getBatchSourceConfig().getDiscoveryTriggererConfig());
    assertEquals(new Gson().toJson(sourceConfig), new Gson().toJson(mergedConfig));
}

Example 2 with BatchSourceConfig

use of org.apache.pulsar.common.io.BatchSourceConfig in project pulsar by apache.

the class SourceConfigUtils method computeBatchSourceIntermediateTopicSubscriptions.

public static Map<String, String> computeBatchSourceIntermediateTopicSubscriptions(Function.FunctionDetails details, String fqfn) {
    Map<String, Object> configMap = extractSourceConfig(details.getSource(), fqfn);
    if (configMap != null) {
        BatchSourceConfig batchSourceConfig = extractBatchSourceConfig(configMap);
        String intermediateTopicName = computeBatchSourceIntermediateTopicName(details.getTenant(), details.getNamespace(), details.getName()).toString();
        if (batchSourceConfig != null) {
            Map<String, String> subscriptionMap = new HashMap<>();
            subscriptionMap.put(intermediateTopicName, computeBatchSourceInstanceSubscriptionName(details.getTenant(), details.getNamespace(), details.getName()));
            return subscriptionMap;
        }
    }
    return null;
}

Example 3 with BatchSourceConfig

use of org.apache.pulsar.common.io.BatchSourceConfig in project pulsar by apache.

the class SourceConfigUtils method convertFromDetails.

public static SourceConfig convertFromDetails(FunctionDetails functionDetails) {
    SourceConfig sourceConfig = new SourceConfig();
    sourceConfig.setTenant(functionDetails.getTenant());
    sourceConfig.setNamespace(functionDetails.getNamespace());
    sourceConfig.setName(functionDetails.getName());
    sourceConfig.setParallelism(functionDetails.getParallelism());
    sourceConfig.setProcessingGuarantees(FunctionCommon.convertProcessingGuarantee(functionDetails.getProcessingGuarantees()));
    Function.SourceSpec sourceSpec = functionDetails.getSource();
    if (!StringUtils.isEmpty(sourceSpec.getClassName())) {
        sourceConfig.setClassName(sourceSpec.getClassName());
    }
    if (!StringUtils.isEmpty(sourceSpec.getBuiltin())) {
        sourceConfig.setArchive("builtin://" + sourceSpec.getBuiltin());
    }
    Map<String, Object> configMap = extractSourceConfig(sourceSpec, FunctionCommon.getFullyQualifiedName(functionDetails));
    if (configMap != null) {
        BatchSourceConfig batchSourceConfig = extractBatchSourceConfig(configMap);
        if (batchSourceConfig != null) {
            sourceConfig.setBatchSourceConfig(batchSourceConfig);
            if (configMap.containsKey(BatchSourceConfig.BATCHSOURCE_CLASSNAME_KEY)) {
                if (!StringUtils.isEmpty((String) configMap.get(BatchSourceConfig.BATCHSOURCE_CLASSNAME_KEY))) {
                    sourceConfig.setClassName((String) configMap.get(BatchSourceConfig.BATCHSOURCE_CLASSNAME_KEY));
                } else {
                    sourceConfig.setClassName(null);
                }
            }
        }
        configMap.remove(BatchSourceConfig.BATCHSOURCE_CONFIG_KEY);
        configMap.remove(BatchSourceConfig.BATCHSOURCE_CLASSNAME_KEY);
        sourceConfig.setConfigs(configMap);
    }
    if (!isEmpty(functionDetails.getSecretsMap())) {
        Type type = new TypeToken<Map<String, Object>>() {
        }.getType();
        Map<String, Object> secretsMap = new Gson().fromJson(functionDetails.getSecretsMap(), type);
        sourceConfig.setSecrets(secretsMap);
    }
    Function.SinkSpec sinkSpec = functionDetails.getSink();
    sourceConfig.setTopicName(sinkSpec.getTopic());
    if (!StringUtils.isEmpty(sinkSpec.getSchemaType())) {
        sourceConfig.setSchemaType(sinkSpec.getSchemaType());
    }
    if (!StringUtils.isEmpty(sinkSpec.getSerDeClassName())) {
        sourceConfig.setSerdeClassName(sinkSpec.getSerDeClassName());
    }
    if (sinkSpec.getProducerSpec() != null) {
        Function.ProducerSpec spec = sinkSpec.getProducerSpec();
        ProducerConfig producerConfig = new ProducerConfig();
        if (spec.getMaxPendingMessages() != 0) {
            producerConfig.setMaxPendingMessages(spec.getMaxPendingMessages());
        }
        if (spec.getMaxPendingMessagesAcrossPartitions() != 0) {
            producerConfig.setMaxPendingMessagesAcrossPartitions(spec.getMaxPendingMessagesAcrossPartitions());
        }
        if (spec.hasCryptoSpec()) {
            producerConfig.setCryptoConfig(CryptoUtils.convertFromSpec(spec.getCryptoSpec()));
        }
        if (spec.getBatchBuilder() != null) {
            producerConfig.setBatchBuilder(spec.getBatchBuilder());
        }
        producerConfig.setUseThreadLocalProducers(spec.getUseThreadLocalProducers());
        sourceConfig.setProducerConfig(producerConfig);
    }
    if (functionDetails.hasResources()) {
        Resources resources = new Resources();
        resources.setCpu(functionDetails.getResources().getCpu());
        resources.setRam(functionDetails.getResources().getRam());
        resources.setDisk(functionDetails.getResources().getDisk());
        sourceConfig.setResources(resources);
    }
    if (!isEmpty(functionDetails.getRuntimeFlags())) {
        sourceConfig.setRuntimeFlags(functionDetails.getRuntimeFlags());
    }
    if (!isEmpty(functionDetails.getCustomRuntimeOptions())) {
        sourceConfig.setCustomRuntimeOptions(functionDetails.getCustomRuntimeOptions());
    }
    return sourceConfig;
}

Example 4 with BatchSourceConfig

use of org.apache.pulsar.common.io.BatchSourceConfig in project pulsar by yahoo.

the class SourceConfigUtilsTest method testMergeDifferentBatchTriggerer.

@Test(expectedExceptions = IllegalArgumentException.class, expectedExceptionsMessageRegExp = "DiscoverTriggerer class cannot be updated for batchsources")
public void testMergeDifferentBatchTriggerer() {
    SourceConfig sourceConfig = createSourceConfigWithBatch();
    BatchSourceConfig batchSourceConfig = createBatchSourceConfig();
    batchSourceConfig.setDiscoveryTriggererClassName("SomeOtherClassName");
    SourceConfig newSourceConfig = createUpdatedSourceConfig("batchSourceConfig", batchSourceConfig);
    SourceConfigUtils.validateUpdate(sourceConfig, newSourceConfig);
}

Example 5 with BatchSourceConfig

use of org.apache.pulsar.common.io.BatchSourceConfig in project pulsar by yahoo.

the class SourceConfigUtilsTest method createSourceConfigWithBatch.

private SourceConfig createSourceConfigWithBatch() {
    SourceConfig sourceConfig = createSourceConfig();
    BatchSourceConfig batchSourceConfig = createBatchSourceConfig();
    sourceConfig.setBatchSourceConfig(batchSourceConfig);
    return sourceConfig;
}