Search in sources :

Example 21 with ConfigInfos

use of org.apache.kafka.connect.runtime.rest.entities.ConfigInfos in project kafka by apache.

the class AbstractHerderTest method testGenerateResultWithConfigValuesAllUsingConfigKeysAndWithSomeErrors.

@Test
public void testGenerateResultWithConfigValuesAllUsingConfigKeysAndWithSomeErrors() {
    String name = "com.acme.connector.MyConnector";
    Map<String, ConfigDef.ConfigKey> keys = new HashMap<>();
    addConfigKey(keys, "config.a1", null);
    addConfigKey(keys, "config.b1", "group B");
    addConfigKey(keys, "config.b2", "group B");
    addConfigKey(keys, "config.c1", "group C");
    List<String> groups = Arrays.asList("groupB", "group C");
    List<ConfigValue> values = new ArrayList<>();
    addValue(values, "config.a1", "value.a1");
    addValue(values, "config.b1", "value.b1");
    addValue(values, "config.b2", "value.b2");
    addValue(values, "config.c1", "value.c1", "error c1");
    ConfigInfos infos = AbstractHerder.generateResult(name, keys, values, groups);
    assertEquals(name, infos.name());
    assertEquals(groups, infos.groups());
    assertEquals(values.size(), infos.values().size());
    assertEquals(1, infos.errorCount());
    assertInfoKey(infos, "config.a1", null);
    assertInfoKey(infos, "config.b1", "group B");
    assertInfoKey(infos, "config.b2", "group B");
    assertInfoKey(infos, "config.c1", "group C");
    assertInfoValue(infos, "config.a1", "value.a1");
    assertInfoValue(infos, "config.b1", "value.b1");
    assertInfoValue(infos, "config.b2", "value.b2");
    assertInfoValue(infos, "config.c1", "value.c1", "error c1");
}
Also used : ConfigValue(org.apache.kafka.common.config.ConfigValue) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ConfigInfos(org.apache.kafka.connect.runtime.rest.entities.ConfigInfos) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 22 with ConfigInfos

use of org.apache.kafka.connect.runtime.rest.entities.ConfigInfos in project kafka by apache.

the class AbstractHerderTest method testConfigValidationTransformsExtendResults.

@Test()
public void testConfigValidationTransformsExtendResults() {
    AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class, noneConnectorClientConfigOverridePolicy);
    // 2 transform aliases defined -> 2 plugin lookups
    Set<PluginDesc<Transformation<?>>> transformations = new HashSet<>();
    transformations.add(transformationPluginDesc());
    EasyMock.expect(plugins.transformations()).andReturn(transformations).times(2);
    replayAll();
    // Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
    // class info that should generate an error.
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSourceConnector.class.getName());
    config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA,xformB");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
    // connector required config
    config.put("required", "value");
    ConfigInfos result = herder.validateConnectorConfig(config, false);
    assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
    // We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
    // the config fields for SourceConnectorConfig, but we expect these to change rarely.
    assertEquals(TestSourceConnector.class.getName(), result.name());
    // Each transform also gets its own group
    List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, "Transforms: xformA", "Transforms: xformB");
    assertEquals(expectedGroups, result.groups());
    assertEquals(2, result.errorCount());
    Map<String, ConfigInfo> infos = result.values().stream().collect(Collectors.toMap(info -> info.configKey().name(), Function.identity()));
    assertEquals(22, infos.size());
    // Should get 2 type fields from the transforms, first adds its own config since it has a valid class
    assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name());
    assertTrue(infos.get("transforms.xformA.type").configValue().errors().isEmpty());
    assertEquals("transforms.xformA.subconfig", infos.get("transforms.xformA.subconfig").configValue().name());
    assertEquals("transforms.xformB.type", infos.get("transforms.xformB.type").configValue().name());
    assertFalse(infos.get("transforms.xformB.type").configValue().errors().isEmpty());
    verifyAll();
}
Also used : Arrays(java.util.Arrays) EasyMock.strictMock(org.easymock.EasyMock.strictMock) Plugins(org.apache.kafka.connect.runtime.isolation.Plugins) Transformation(org.apache.kafka.connect.transforms.Transformation) MockStrict(org.powermock.api.easymock.annotation.MockStrict) SourceConnector(org.apache.kafka.connect.source.SourceConnector) AllConnectorClientConfigOverridePolicy(org.apache.kafka.connect.connector.policy.AllConnectorClientConfigOverridePolicy) Map(java.util.Map) ConnectorType(org.apache.kafka.connect.runtime.rest.entities.ConnectorType) PowerMock.replayAll(org.powermock.api.easymock.PowerMock.replayAll) ConfigDef(org.apache.kafka.common.config.ConfigDef) PrincipalConnectorClientConfigOverridePolicy(org.apache.kafka.connect.connector.policy.PrincipalConnectorClientConfigOverridePolicy) CommonClientConfigs(org.apache.kafka.clients.CommonClientConfigs) Collection(java.util.Collection) ConnectRecord(org.apache.kafka.connect.connector.ConnectRecord) Set(java.util.Set) ConfigValue(org.apache.kafka.common.config.ConfigValue) PowerMock(org.powermock.api.easymock.PowerMock) Collectors(java.util.stream.Collectors) EasyMock.partialMockBuilder(org.easymock.EasyMock.partialMockBuilder) ConfigValueInfo(org.apache.kafka.connect.runtime.rest.entities.ConfigValueInfo) ConnectorStateInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo) ConfigInfo(org.apache.kafka.connect.runtime.rest.entities.ConfigInfo) List(java.util.List) Predicate(org.apache.kafka.connect.transforms.predicates.Predicate) ConfigInfos(org.apache.kafka.connect.runtime.rest.entities.ConfigInfos) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) BadRequestException(org.apache.kafka.connect.runtime.rest.errors.BadRequestException) ConnectorClientConfigOverridePolicy(org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy) SourceTask(org.apache.kafka.connect.source.SourceTask) Connector(org.apache.kafka.connect.connector.Connector) ConnectorTaskId(org.apache.kafka.connect.util.ConnectorTaskId) Assert.assertThrows(org.junit.Assert.assertThrows) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) ConfigBackingStore(org.apache.kafka.connect.storage.ConfigBackingStore) Function(java.util.function.Function) NoneConnectorClientConfigOverridePolicy(org.apache.kafka.connect.connector.policy.NoneConnectorClientConfigOverridePolicy) PluginDesc(org.apache.kafka.connect.runtime.isolation.PluginDesc) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) PowerMock.verifyAll(org.powermock.api.easymock.PowerMock.verifyAll) SaslConfigs(org.apache.kafka.common.config.SaslConfigs) PowerMockRunner(org.powermock.modules.junit4.PowerMockRunner) ProducerConfig(org.apache.kafka.clients.producer.ProducerConfig) Capture(org.easymock.Capture) AbstractHerder.keysWithVariableValues(org.apache.kafka.connect.runtime.AbstractHerder.keysWithVariableValues) Assert.assertNotNull(org.junit.Assert.assertNotNull) ClusterConfigState(org.apache.kafka.connect.runtime.distributed.ClusterConfigState) ConfigTransformer(org.apache.kafka.common.config.ConfigTransformer) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) EasyMock(org.easymock.EasyMock) ConfigException(org.apache.kafka.common.config.ConfigException) StatusBackingStore(org.apache.kafka.connect.storage.StatusBackingStore) OAuthBearerUnsecuredLoginCallbackHandler(org.apache.kafka.common.security.oauthbearer.internals.unsecured.OAuthBearerUnsecuredLoginCallbackHandler) Assert.assertNull(org.junit.Assert.assertNull) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) HashMap(java.util.HashMap) ConfigInfo(org.apache.kafka.connect.runtime.rest.entities.ConfigInfo) PluginDesc(org.apache.kafka.connect.runtime.isolation.PluginDesc) HashSet(java.util.HashSet) ConfigInfos(org.apache.kafka.connect.runtime.rest.entities.ConfigInfos) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 23 with ConfigInfos

use of org.apache.kafka.connect.runtime.rest.entities.ConfigInfos in project kafka by apache.

the class AbstractHerderTest method testConfigValidationPredicatesExtendResults.

@Test()
public void testConfigValidationPredicatesExtendResults() {
    AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class, noneConnectorClientConfigOverridePolicy);
    // 2 transform aliases defined -> 2 plugin lookups
    Set<PluginDesc<Transformation<?>>> transformations = new HashSet<>();
    transformations.add(transformationPluginDesc());
    EasyMock.expect(plugins.transformations()).andReturn(transformations).times(1);
    Set<PluginDesc<Predicate<?>>> predicates = new HashSet<>();
    predicates.add(predicatePluginDesc());
    EasyMock.expect(plugins.predicates()).andReturn(predicates).times(2);
    replayAll();
    // Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
    // class info that should generate an error.
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSourceConnector.class.getName());
    config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.predicate", "predX");
    config.put(ConnectorConfig.PREDICATES_CONFIG, "predX,predY");
    config.put(ConnectorConfig.PREDICATES_CONFIG + ".predX.type", SamplePredicate.class.getName());
    // connector required config
    config.put("required", "value");
    ConfigInfos result = herder.validateConnectorConfig(config, false);
    assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
    // We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
    // the config fields for SourceConnectorConfig, but we expect these to change rarely.
    assertEquals(TestSourceConnector.class.getName(), result.name());
    // Each transform also gets its own group
    List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, "Transforms: xformA", "Predicates: predX", "Predicates: predY");
    assertEquals(expectedGroups, result.groups());
    assertEquals(2, result.errorCount());
    Map<String, ConfigInfo> infos = result.values().stream().collect(Collectors.toMap(info -> info.configKey().name(), Function.identity()));
    assertEquals(24, infos.size());
    // Should get 2 type fields from the transforms, first adds its own config since it has a valid class
    assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name());
    assertTrue(infos.get("transforms.xformA.type").configValue().errors().isEmpty());
    assertEquals("transforms.xformA.subconfig", infos.get("transforms.xformA.subconfig").configValue().name());
    assertEquals("transforms.xformA.predicate", infos.get("transforms.xformA.predicate").configValue().name());
    assertTrue(infos.get("transforms.xformA.predicate").configValue().errors().isEmpty());
    assertEquals("transforms.xformA.negate", infos.get("transforms.xformA.negate").configValue().name());
    assertTrue(infos.get("transforms.xformA.negate").configValue().errors().isEmpty());
    assertEquals("predicates.predX.type", infos.get("predicates.predX.type").configValue().name());
    assertEquals("predicates.predX.predconfig", infos.get("predicates.predX.predconfig").configValue().name());
    assertEquals("predicates.predY.type", infos.get("predicates.predY.type").configValue().name());
    assertFalse(infos.get("predicates.predY.type").configValue().errors().isEmpty());
    verifyAll();
}
Also used : Arrays(java.util.Arrays) EasyMock.strictMock(org.easymock.EasyMock.strictMock) Plugins(org.apache.kafka.connect.runtime.isolation.Plugins) Transformation(org.apache.kafka.connect.transforms.Transformation) MockStrict(org.powermock.api.easymock.annotation.MockStrict) SourceConnector(org.apache.kafka.connect.source.SourceConnector) AllConnectorClientConfigOverridePolicy(org.apache.kafka.connect.connector.policy.AllConnectorClientConfigOverridePolicy) Map(java.util.Map) ConnectorType(org.apache.kafka.connect.runtime.rest.entities.ConnectorType) PowerMock.replayAll(org.powermock.api.easymock.PowerMock.replayAll) ConfigDef(org.apache.kafka.common.config.ConfigDef) PrincipalConnectorClientConfigOverridePolicy(org.apache.kafka.connect.connector.policy.PrincipalConnectorClientConfigOverridePolicy) CommonClientConfigs(org.apache.kafka.clients.CommonClientConfigs) Collection(java.util.Collection) ConnectRecord(org.apache.kafka.connect.connector.ConnectRecord) Set(java.util.Set) ConfigValue(org.apache.kafka.common.config.ConfigValue) PowerMock(org.powermock.api.easymock.PowerMock) Collectors(java.util.stream.Collectors) EasyMock.partialMockBuilder(org.easymock.EasyMock.partialMockBuilder) ConfigValueInfo(org.apache.kafka.connect.runtime.rest.entities.ConfigValueInfo) ConnectorStateInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo) ConfigInfo(org.apache.kafka.connect.runtime.rest.entities.ConfigInfo) List(java.util.List) Predicate(org.apache.kafka.connect.transforms.predicates.Predicate) ConfigInfos(org.apache.kafka.connect.runtime.rest.entities.ConfigInfos) Assert.assertFalse(org.junit.Assert.assertFalse) Optional(java.util.Optional) BadRequestException(org.apache.kafka.connect.runtime.rest.errors.BadRequestException) ConnectorClientConfigOverridePolicy(org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy) SourceTask(org.apache.kafka.connect.source.SourceTask) Connector(org.apache.kafka.connect.connector.Connector) ConnectorTaskId(org.apache.kafka.connect.util.ConnectorTaskId) Assert.assertThrows(org.junit.Assert.assertThrows) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) ConfigBackingStore(org.apache.kafka.connect.storage.ConfigBackingStore) Function(java.util.function.Function) NoneConnectorClientConfigOverridePolicy(org.apache.kafka.connect.connector.policy.NoneConnectorClientConfigOverridePolicy) PluginDesc(org.apache.kafka.connect.runtime.isolation.PluginDesc) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) PowerMock.verifyAll(org.powermock.api.easymock.PowerMock.verifyAll) SaslConfigs(org.apache.kafka.common.config.SaslConfigs) PowerMockRunner(org.powermock.modules.junit4.PowerMockRunner) ProducerConfig(org.apache.kafka.clients.producer.ProducerConfig) Capture(org.easymock.Capture) AbstractHerder.keysWithVariableValues(org.apache.kafka.connect.runtime.AbstractHerder.keysWithVariableValues) Assert.assertNotNull(org.junit.Assert.assertNotNull) ClusterConfigState(org.apache.kafka.connect.runtime.distributed.ClusterConfigState) ConfigTransformer(org.apache.kafka.common.config.ConfigTransformer) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) EasyMock(org.easymock.EasyMock) ConfigException(org.apache.kafka.common.config.ConfigException) StatusBackingStore(org.apache.kafka.connect.storage.StatusBackingStore) OAuthBearerUnsecuredLoginCallbackHandler(org.apache.kafka.common.security.oauthbearer.internals.unsecured.OAuthBearerUnsecuredLoginCallbackHandler) Assert.assertNull(org.junit.Assert.assertNull) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) HashMap(java.util.HashMap) ConfigInfo(org.apache.kafka.connect.runtime.rest.entities.ConfigInfo) PluginDesc(org.apache.kafka.connect.runtime.isolation.PluginDesc) HashSet(java.util.HashSet) ConfigInfos(org.apache.kafka.connect.runtime.rest.entities.ConfigInfos) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 24 with ConfigInfos

use of org.apache.kafka.connect.runtime.rest.entities.ConfigInfos in project kafka by apache.

the class AbstractHerderTest method testGenerateResultWithConfigValuesWithNoConfigKeysAndWithSomeErrors.

@Test
public void testGenerateResultWithConfigValuesWithNoConfigKeysAndWithSomeErrors() {
    String name = "com.acme.connector.MyConnector";
    Map<String, ConfigDef.ConfigKey> keys = new HashMap<>();
    List<String> groups = new ArrayList<>();
    List<ConfigValue> values = new ArrayList<>();
    addValue(values, "config.a1", "value.a1");
    addValue(values, "config.b1", "value.b1");
    addValue(values, "config.b2", "value.b2");
    addValue(values, "config.c1", "value.c1", "error c1");
    addValue(values, "config.extra1", "value.extra1");
    addValue(values, "config.extra2", "value.extra2", "error extra2");
    ConfigInfos infos = AbstractHerder.generateResult(name, keys, values, groups);
    assertEquals(name, infos.name());
    assertEquals(groups, infos.groups());
    assertEquals(values.size(), infos.values().size());
    assertEquals(2, infos.errorCount());
    assertNoInfoKey(infos, "config.a1");
    assertNoInfoKey(infos, "config.b1");
    assertNoInfoKey(infos, "config.b2");
    assertNoInfoKey(infos, "config.c1");
    assertNoInfoKey(infos, "config.extra1");
    assertNoInfoKey(infos, "config.extra2");
    assertInfoValue(infos, "config.a1", "value.a1");
    assertInfoValue(infos, "config.b1", "value.b1");
    assertInfoValue(infos, "config.b2", "value.b2");
    assertInfoValue(infos, "config.c1", "value.c1", "error c1");
    assertInfoValue(infos, "config.extra1", "value.extra1");
    assertInfoValue(infos, "config.extra2", "value.extra2", "error extra2");
}
Also used : ConfigValue(org.apache.kafka.common.config.ConfigValue) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ConfigInfos(org.apache.kafka.connect.runtime.rest.entities.ConfigInfos) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 25 with ConfigInfos

use of org.apache.kafka.connect.runtime.rest.entities.ConfigInfos in project kafka by apache.

the class AbstractHerderTest method testConfigValidationNullConfig.

@Test()
public void testConfigValidationNullConfig() {
    AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class, noneConnectorClientConfigOverridePolicy);
    replayAll();
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSourceConnector.class.getName());
    config.put("name", "somename");
    config.put("required", "value");
    config.put("testKey", null);
    final ConfigInfos configInfos = herder.validateConnectorConfig(config, false);
    assertEquals(1, configInfos.errorCount());
    assertErrorForKey(configInfos, "testKey");
    verifyAll();
}
Also used : HashMap(java.util.HashMap) ConfigInfos(org.apache.kafka.connect.runtime.rest.entities.ConfigInfos) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

ConfigInfos (org.apache.kafka.connect.runtime.rest.entities.ConfigInfos)29 HashMap (java.util.HashMap)23 Test (org.junit.Test)20 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)20 Map (java.util.Map)16 ConfigDef (org.apache.kafka.common.config.ConfigDef)13 ConfigValue (org.apache.kafka.common.config.ConfigValue)13 Connector (org.apache.kafka.connect.connector.Connector)13 SourceConnector (org.apache.kafka.connect.source.SourceConnector)13 List (java.util.List)12 LinkedList (java.util.LinkedList)11 ArrayList (java.util.ArrayList)10 ConfigInfo (org.apache.kafka.connect.runtime.rest.entities.ConfigInfo)9 Config (org.apache.kafka.common.config.Config)8 ConnectorConfig (org.apache.kafka.connect.runtime.ConnectorConfig)7 BadRequestException (org.apache.kafka.connect.runtime.rest.errors.BadRequestException)7 SinkConnector (org.apache.kafka.connect.sink.SinkConnector)7 MockConnector (org.apache.kafka.connect.tools.MockConnector)7 MockSinkConnector (org.apache.kafka.connect.tools.MockSinkConnector)7 MockSourceConnector (org.apache.kafka.connect.tools.MockSourceConnector)7