Search in sources :

Example 6 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project kafka by apache.

the class Connector method validate.

/**
     * Validate the connector configuration values against configuration definitions.
     * @param connectorConfigs the provided configuration values
     * @return List of Config, each Config contains the updated configuration information given
     * the current configuration values.
     */
public Config validate(Map<String, String> connectorConfigs) {
    ConfigDef configDef = config();
    List<ConfigValue> configValues = configDef.validate(connectorConfigs);
    return new Config(configValues);
}
Also used : ConfigValue(org.apache.kafka.common.config.ConfigValue) Config(org.apache.kafka.common.config.Config) ConfigDef(org.apache.kafka.common.config.ConfigDef)

Example 7 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project kafka by apache.

the class AbstractHerder method validateConnectorConfig.

@Override
public ConfigInfos validateConnectorConfig(Map<String, String> connectorConfig) {
    String connType = connectorConfig.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
    if (connType == null)
        throw new BadRequestException("Connector config " + connectorConfig + " contains no connector type");
    Connector connector = getConnector(connType);
    final ConfigDef connectorConfigDef = ConnectorConfig.enrich((connector instanceof SourceConnector) ? SourceConnectorConfig.configDef() : SinkConnectorConfig.configDef(), connectorConfig, false);
    List<ConfigValue> configValues = new ArrayList<>();
    Map<String, ConfigKey> configKeys = new HashMap<>();
    List<String> allGroups = new ArrayList<>();
    // do basic connector validation (name, connector type, etc.)
    Map<String, ConfigValue> validatedConnectorConfig = validateBasicConnectorConfig(connector, connectorConfigDef, connectorConfig);
    configValues.addAll(validatedConnectorConfig.values());
    configKeys.putAll(connectorConfigDef.configKeys());
    allGroups.addAll(connectorConfigDef.groups());
    // do custom connector-specific validation
    Config config = connector.validate(connectorConfig);
    ConfigDef configDef = connector.config();
    configKeys.putAll(configDef.configKeys());
    allGroups.addAll(configDef.groups());
    configValues.addAll(config.configValues());
    return generateResult(connType, configKeys, configValues, allGroups);
}
Also used : Connector(org.apache.kafka.connect.connector.Connector) SourceConnector(org.apache.kafka.connect.source.SourceConnector) ConfigValue(org.apache.kafka.common.config.ConfigValue) ConfigKey(org.apache.kafka.common.config.ConfigDef.ConfigKey) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Config(org.apache.kafka.common.config.Config) ArrayList(java.util.ArrayList) SourceConnector(org.apache.kafka.connect.source.SourceConnector) BadRequestException(org.apache.kafka.connect.runtime.rest.errors.BadRequestException) ConfigDef(org.apache.kafka.common.config.ConfigDef)

Example 8 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project kafka by apache.

the class StandaloneHerderTest method expectConfigValidation.

private void expectConfigValidation(Map<String, String>... configs) {
    // config validation
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andStubReturn(new ConfigDef());
    for (Map<String, String> config : configs) EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConnectorFactory(org.apache.kafka.connect.runtime.ConnectorFactory) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) Config(org.apache.kafka.common.config.Config) TaskConfig(org.apache.kafka.connect.runtime.TaskConfig) ConfigDef(org.apache.kafka.common.config.ConfigDef)

Example 9 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project kafka by apache.

the class StandaloneHerderTest method testCreateConnectorFailedCustomValidation.

@Test
public void testCreateConnectorFailedCustomValidation() throws Exception {
    connector = PowerMock.createMock(BogusSourceConnector.class);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    ConfigDef configDef = new ConfigDef();
    configDef.define("foo.bar", ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, "foo.bar doc");
    EasyMock.expect(connectorMock.config()).andReturn(configDef);
    ConfigValue validatedValue = new ConfigValue("foo.bar");
    validatedValue.addErrorMessage("Failed foo.bar validation");
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(singletonList(validatedValue)));
    createCallback.onCompletion(EasyMock.<BadRequestException>anyObject(), EasyMock.<Herder.Created<ConnectorInfo>>isNull());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConfigValue(org.apache.kafka.common.config.ConfigValue) ConnectorFactory(org.apache.kafka.connect.runtime.ConnectorFactory) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) Config(org.apache.kafka.common.config.Config) TaskConfig(org.apache.kafka.connect.runtime.TaskConfig) ConfigDef(org.apache.kafka.common.config.ConfigDef) Herder(org.apache.kafka.connect.runtime.Herder) Test(org.junit.Test)

Example 10 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project kafka by apache.

the class ConnectorPluginsResourceTest method testValidateConfig.

@Test
public void testValidateConfig() throws Throwable {
    herder.validateConnectorConfig(EasyMock.eq(props));
    PowerMock.expectLastCall().andAnswer(new IAnswer<ConfigInfos>() {

        @Override
        public ConfigInfos answer() {
            ConfigDef connectorConfigDef = ConnectorConfig.configDef();
            List<ConfigValue> connectorConfigValues = connectorConfigDef.validate(props);
            Connector connector = new ConnectorPluginsResourceTestConnector();
            Config config = connector.validate(props);
            ConfigDef configDef = connector.config();
            Map<String, ConfigDef.ConfigKey> configKeys = configDef.configKeys();
            List<ConfigValue> configValues = config.configValues();
            Map<String, ConfigDef.ConfigKey> resultConfigKeys = new HashMap<>(configKeys);
            resultConfigKeys.putAll(connectorConfigDef.configKeys());
            configValues.addAll(connectorConfigValues);
            return AbstractHerder.generateResult(ConnectorPluginsResourceTestConnector.class.getName(), resultConfigKeys, configValues, Collections.singletonList("Test"));
        }
    });
    PowerMock.replayAll();
    ConfigInfos configInfos = connectorPluginsResource.validateConfigs(ConnectorPluginsResourceTestConnector.class.getName(), props);
    assertEquals(CONFIG_INFOS.name(), configInfos.name());
    assertEquals(CONFIG_INFOS.errorCount(), configInfos.errorCount());
    assertEquals(CONFIG_INFOS.groups(), configInfos.groups());
    assertEquals(new HashSet<>(CONFIG_INFOS.values()), new HashSet<>(configInfos.values()));
    PowerMock.verifyAll();
}
Also used : Connector(org.apache.kafka.connect.connector.Connector) SchemaSourceConnector(org.apache.kafka.connect.tools.SchemaSourceConnector) SourceConnector(org.apache.kafka.connect.source.SourceConnector) VerifiableSourceConnector(org.apache.kafka.connect.tools.VerifiableSourceConnector) MockSinkConnector(org.apache.kafka.connect.tools.MockSinkConnector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) MockSourceConnector(org.apache.kafka.connect.tools.MockSourceConnector) VerifiableSinkConnector(org.apache.kafka.connect.tools.VerifiableSinkConnector) MockConnector(org.apache.kafka.connect.tools.MockConnector) Config(org.apache.kafka.common.config.Config) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) LinkedList(java.util.LinkedList) List(java.util.List) ConfigDef(org.apache.kafka.common.config.ConfigDef) HashMap(java.util.HashMap) Map(java.util.Map) ConfigInfos(org.apache.kafka.connect.runtime.rest.entities.ConfigInfos) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

ConfigDef (org.apache.kafka.common.config.ConfigDef)12 Config (org.apache.kafka.common.config.Config)11 Connector (org.apache.kafka.connect.connector.Connector)10 SourceConnector (org.apache.kafka.connect.source.SourceConnector)10 ConfigValue (org.apache.kafka.common.config.ConfigValue)9 ConnectorConfig (org.apache.kafka.connect.runtime.ConnectorConfig)9 SinkConnector (org.apache.kafka.connect.sink.SinkConnector)9 ConnectorFactory (org.apache.kafka.connect.runtime.ConnectorFactory)8 TaskConfig (org.apache.kafka.connect.runtime.TaskConfig)8 Test (org.junit.Test)8 Herder (org.apache.kafka.connect.runtime.Herder)6 ConnectorInfo (org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo)6 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)6 HashMap (java.util.HashMap)5 SinkConnectorConfig (org.apache.kafka.connect.runtime.SinkConnectorConfig)5 WorkerConfig (org.apache.kafka.connect.runtime.WorkerConfig)5 BadRequestException (org.apache.kafka.connect.runtime.rest.errors.BadRequestException)4 ArrayList (java.util.ArrayList)2 List (java.util.List)2 Map (java.util.Map)2