Search in sources :

Example 36 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project kafka by apache.

the class ConnectorPluginsResourceTest method testValidateConfigWithSingleErrorDueToMissingConnectorClassname.

@Test
public void testValidateConfigWithSingleErrorDueToMissingConnectorClassname() throws Throwable {
    Capture<Callback<ConfigInfos>> configInfosCallback = EasyMock.newCapture();
    herder.validateConnectorConfig(EasyMock.eq(partialProps), EasyMock.capture(configInfosCallback), EasyMock.anyBoolean());
    PowerMock.expectLastCall().andAnswer((IAnswer<Void>) () -> {
        ConfigDef connectorConfigDef = ConnectorConfig.configDef();
        List<ConfigValue> connectorConfigValues = connectorConfigDef.validate(partialProps);
        Connector connector = new ConnectorPluginsResourceTestConnector();
        Config config = connector.validate(partialProps);
        ConfigDef configDef = connector.config();
        Map<String, ConfigDef.ConfigKey> configKeys = configDef.configKeys();
        List<ConfigValue> configValues = config.configValues();
        Map<String, ConfigDef.ConfigKey> resultConfigKeys = new HashMap<>(configKeys);
        resultConfigKeys.putAll(connectorConfigDef.configKeys());
        configValues.addAll(connectorConfigValues);
        ConfigInfos configInfos = AbstractHerder.generateResult(ConnectorPluginsResourceTestConnector.class.getName(), resultConfigKeys, configValues, Collections.singletonList("Test"));
        configInfosCallback.getValue().onCompletion(null, configInfos);
        return null;
    });
    PowerMock.replayAll();
    // This call to validateConfigs does not throw a BadRequestException because we've mocked
    // validateConnectorConfig.
    ConfigInfos configInfos = connectorPluginsResource.validateConfigs(ConnectorPluginsResourceTestConnector.class.getSimpleName(), partialProps);
    assertEquals(PARTIAL_CONFIG_INFOS.name(), configInfos.name());
    assertEquals(PARTIAL_CONFIG_INFOS.errorCount(), configInfos.errorCount());
    assertEquals(PARTIAL_CONFIG_INFOS.groups(), configInfos.groups());
    assertEquals(new HashSet<>(PARTIAL_CONFIG_INFOS.values()), new HashSet<>(configInfos.values()));
    PowerMock.verifyAll();
}
Also used : TestSourceConnector(org.apache.kafka.connect.runtime.TestSourceConnector) SourceConnector(org.apache.kafka.connect.source.SourceConnector) VerifiableSourceConnector(org.apache.kafka.connect.tools.VerifiableSourceConnector) MockSinkConnector(org.apache.kafka.connect.tools.MockSinkConnector) MockSourceConnector(org.apache.kafka.connect.tools.MockSourceConnector) VerifiableSinkConnector(org.apache.kafka.connect.tools.VerifiableSinkConnector) MockConnector(org.apache.kafka.connect.tools.MockConnector) Connector(org.apache.kafka.connect.connector.Connector) SchemaSourceConnector(org.apache.kafka.connect.tools.SchemaSourceConnector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) TestSinkConnector(org.apache.kafka.connect.runtime.TestSinkConnector) Callback(org.apache.kafka.connect.util.Callback) WorkerConfig(org.apache.kafka.connect.runtime.WorkerConfig) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) Config(org.apache.kafka.common.config.Config) Arrays.asList(java.util.Arrays.asList) List(java.util.List) LinkedList(java.util.LinkedList) ConfigDef(org.apache.kafka.common.config.ConfigDef) Map(java.util.Map) HashMap(java.util.HashMap) ConfigInfos(org.apache.kafka.connect.runtime.rest.entities.ConfigInfos) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 37 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project kafka by apache.

the class StandaloneHerderTest method testCreateConnectorFailedValidation.

@Test
public void testCreateConnectorFailedValidation() {
    // Basic validation should be performed and return an error, but should still evaluate the connector's config
    connector = PowerMock.createMock(BogusSourceConnector.class);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    config.remove(ConnectorConfig.NAME_CONFIG);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    EasyMock.expect(worker.configTransformer()).andReturn(transformer).times(2);
    final Capture<Map<String, String>> configCapture = EasyMock.newCapture();
    EasyMock.expect(transformer.transform(EasyMock.capture(configCapture))).andAnswer(configCapture::getValue);
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
    EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
    EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andStubReturn(new ConfigDef());
    ConfigValue validatedValue = new ConfigValue("foo.bar");
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(singletonList(validatedValue)));
    EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    ExecutionException exception = assertThrows(ExecutionException.class, () -> createCallback.get(1000L, TimeUnit.SECONDS));
    assertEquals(BadRequestException.class, exception.getCause().getClass());
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) WorkerConnector(org.apache.kafka.connect.runtime.WorkerConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConfigValue(org.apache.kafka.common.config.ConfigValue) SourceConnectorConfig(org.apache.kafka.connect.runtime.SourceConnectorConfig) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) Config(org.apache.kafka.common.config.Config) SinkConnectorConfig(org.apache.kafka.connect.runtime.SinkConnectorConfig) TaskConfig(org.apache.kafka.connect.runtime.TaskConfig) ExecutionException(java.util.concurrent.ExecutionException) Map(java.util.Map) HashMap(java.util.HashMap) Collections.singletonMap(java.util.Collections.singletonMap) ConfigDef(org.apache.kafka.common.config.ConfigDef) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 38 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project connect-utils by jcustenborder.

the class PluginLoader method config.

Plugin.Configuration config(Class<?> pluginClass) {
    ConfigDef config = configDef(pluginClass);
    if (null == config) {
        return null;
    }
    ImmutableConfiguration.Builder configBuilder = ImmutableConfiguration.builder().configDef(config);
    Map<String, ImmutableGroup.Builder> groupBuilderCache = new LinkedHashMap<>();
    List<ConfigDef.ConfigKey> configKeys = config.configKeys().values().stream().sorted(ConfigKeyComparator.INSTANCE).collect(Collectors.toList());
    for (ConfigDef.ConfigKey configKey : configKeys) {
        final String group = Strings.isNullOrEmpty(configKey.group) ? GENERAL_GROUP : configKey.group;
        Plugin.Item item = ImmutableItem.builder().defaultValue(ConfigDef.NO_DEFAULT_VALUE.equals(configKey.defaultValue) ? null : configKey.defaultValue).doc(configKey.documentation).group(group).importance(configKey.importance).isRequired(!configKey.hasDefault()).name(configKey.name).type(configKey.type).validator(configKey.validator).build();
        ImmutableGroup.Builder groupBuilder = groupBuilderCache.computeIfAbsent(group, s -> ImmutableGroup.builder().name(group));
        groupBuilder.addItems(item);
        if (item.isRequired()) {
            configBuilder.addRequiredConfigs(item);
        }
    }
    groupBuilderCache.values().stream().map(ImmutableGroup.Builder::build).forEach(configBuilder::addGroups);
    return configBuilder.build();
}
Also used : ConfigurationBuilder(org.reflections.util.ConfigurationBuilder) LinkedHashMap(java.util.LinkedHashMap) ConfigDef(org.apache.kafka.common.config.ConfigDef)

Example 39 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project connect-utils by jcustenborder.

the class PluginLoader method configDef.

// ConfigDef transformationConfig(Class<? extends Transformation> transformation) {
// try {
// return transformation.newInstance().config();
// } catch (InstantiationException | IllegalAccessException e) {
// throw new IllegalStateException(e);
// }
// }
// 
// ConfigDef connectorConfig(Class<? extends Connector> connectorClass) {
// try {
// return connectorClass.newInstance().config();
// } catch (InstantiationException | IllegalAccessException e) {
// throw new IllegalStateException(e);
// }
// }
ConfigDef configDef(Class<?> pluginClass) {
    ConfigDef result;
    try {
        if (Connector.class.isAssignableFrom(pluginClass)) {
            Connector connector = (Connector) pluginClass.newInstance();
            result = connector.config();
        } else if (Transformation.class.isAssignableFrom(pluginClass)) {
            Transformation<?> connector = (Transformation<?>) pluginClass.newInstance();
            result = connector.config();
        } else if (Converter.class.isAssignableFrom(pluginClass) || ConfigProvider.class.isAssignableFrom(pluginClass)) {
            Optional<Method> optionalConfigMethod = Stream.of(pluginClass.getMethods()).filter(method -> Modifier.isPublic(method.getModifiers())).filter(method -> ConfigDef.class.equals(method.getReturnType())).findFirst();
            if (optionalConfigMethod.isPresent()) {
                Method configMethod = optionalConfigMethod.get();
                Object instance;
                if (Modifier.isStatic(configMethod.getModifiers())) {
                    instance = null;
                } else {
                    instance = pluginClass.newInstance();
                }
                result = (ConfigDef) configMethod.invoke(instance);
            } else {
                result = null;
            }
        } else {
            throw new UnsupportedOperationException(String.format("Type %s is not supported", pluginClass.getName()));
        }
    } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
        throw new IllegalStateException(e);
    }
    return result;
}
Also used : Connector(org.apache.kafka.connect.connector.Connector) Arrays(java.util.Arrays) ConfigProvider(org.apache.kafka.common.config.provider.ConfigProvider) LoggerFactory(org.slf4j.LoggerFactory) Reflections(org.reflections.Reflections) ArrayList(java.util.ArrayList) Transformation(org.apache.kafka.connect.transforms.Transformation) LinkedHashMap(java.util.LinkedHashMap) Strings(com.google.common.base.Strings) SourceConnector(org.apache.kafka.connect.source.SourceConnector) Converter(org.apache.kafka.connect.storage.Converter) Map(java.util.Map) Method(java.lang.reflect.Method) ConfigurationBuilder(org.reflections.util.ConfigurationBuilder) LinkedHashSet(java.util.LinkedHashSet) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConfigDef(org.apache.kafka.common.config.ConfigDef) ImmutableSet(com.google.common.collect.ImmutableSet) Logger(org.slf4j.Logger) ConfigKeyComparator(com.github.jcustenborder.kafka.connect.utils.config.ConfigKeyComparator) Set(java.util.Set) Collectors(java.util.stream.Collectors) ClasspathHelper(org.reflections.util.ClasspathHelper) InvocationTargetException(java.lang.reflect.InvocationTargetException) ResourcesScanner(org.reflections.scanners.ResourcesScanner) List(java.util.List) Stream(java.util.stream.Stream) Modifier(java.lang.reflect.Modifier) Optional(java.util.Optional) Comparator(java.util.Comparator) Connector(org.apache.kafka.connect.connector.Connector) SourceConnector(org.apache.kafka.connect.source.SourceConnector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) Transformation(org.apache.kafka.connect.transforms.Transformation) ConfigProvider(org.apache.kafka.common.config.provider.ConfigProvider) Method(java.lang.reflect.Method) InvocationTargetException(java.lang.reflect.InvocationTargetException) Converter(org.apache.kafka.connect.storage.Converter) ConfigDef(org.apache.kafka.common.config.ConfigDef)

Example 40 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project kafka by apache.

the class AbstractHerder method validateConnectorConfig.

@Override
public ConfigInfos validateConnectorConfig(Map<String, String> connectorConfig) {
    String connType = connectorConfig.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
    if (connType == null)
        throw new BadRequestException("Connector config " + connectorConfig + " contains no connector type");
    Connector connector = getConnector(connType);
    final ConfigDef connectorConfigDef = ConnectorConfig.enrich((connector instanceof SourceConnector) ? SourceConnectorConfig.configDef() : SinkConnectorConfig.configDef(), connectorConfig, false);
    List<ConfigValue> configValues = new ArrayList<>();
    Map<String, ConfigKey> configKeys = new HashMap<>();
    List<String> allGroups = new ArrayList<>();
    // do basic connector validation (name, connector type, etc.)
    Map<String, ConfigValue> validatedConnectorConfig = validateBasicConnectorConfig(connector, connectorConfigDef, connectorConfig);
    configValues.addAll(validatedConnectorConfig.values());
    configKeys.putAll(connectorConfigDef.configKeys());
    allGroups.addAll(connectorConfigDef.groups());
    // do custom connector-specific validation
    Config config = connector.validate(connectorConfig);
    ConfigDef configDef = connector.config();
    configKeys.putAll(configDef.configKeys());
    allGroups.addAll(configDef.groups());
    configValues.addAll(config.configValues());
    return generateResult(connType, configKeys, configValues, allGroups);
}
Also used : Connector(org.apache.kafka.connect.connector.Connector) SourceConnector(org.apache.kafka.connect.source.SourceConnector) ConfigValue(org.apache.kafka.common.config.ConfigValue) ConfigKey(org.apache.kafka.common.config.ConfigDef.ConfigKey) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Config(org.apache.kafka.common.config.Config) ArrayList(java.util.ArrayList) SourceConnector(org.apache.kafka.connect.source.SourceConnector) BadRequestException(org.apache.kafka.connect.runtime.rest.errors.BadRequestException) ConfigDef(org.apache.kafka.common.config.ConfigDef)

Aggregations

ConfigDef (org.apache.kafka.common.config.ConfigDef)69 Config (org.apache.kafka.common.config.Config)29 Test (org.junit.Test)28 Connector (org.apache.kafka.connect.connector.Connector)27 SourceConnector (org.apache.kafka.connect.source.SourceConnector)26 ConnectorConfig (org.apache.kafka.connect.runtime.ConnectorConfig)24 SinkConnector (org.apache.kafka.connect.sink.SinkConnector)23 HashMap (java.util.HashMap)20 ConfigValue (org.apache.kafka.common.config.ConfigValue)19 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)19 TaskConfig (org.apache.kafka.connect.runtime.TaskConfig)17 Map (java.util.Map)14 SinkConnectorConfig (org.apache.kafka.connect.runtime.SinkConnectorConfig)14 WorkerConfig (org.apache.kafka.connect.runtime.WorkerConfig)13 List (java.util.List)12 ArrayList (java.util.ArrayList)10 Herder (org.apache.kafka.connect.runtime.Herder)10 ConnectorInfo (org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo)10 BadRequestException (org.apache.kafka.connect.runtime.rest.errors.BadRequestException)10 AbstractConfig (org.apache.kafka.common.config.AbstractConfig)9