Examples with ConfigValue org.apache.kafka.common.config.ConfigValue used on opensource projects

Example 6 with ConfigValue

use of org.apache.kafka.common.config.ConfigValue in project kafka by apache.

the class AbstractHerder method validateConnectorConfig.

@Override
public ConfigInfos validateConnectorConfig(Map<String, String> connectorConfig) {
    String connType = connectorConfig.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
    if (connType == null)
        throw new BadRequestException("Connector config " + connectorConfig + " contains no connector type");
    Connector connector = getConnector(connType);
    final ConfigDef connectorConfigDef = ConnectorConfig.enrich((connector instanceof SourceConnector) ? SourceConnectorConfig.configDef() : SinkConnectorConfig.configDef(), connectorConfig, false);
    List<ConfigValue> configValues = new ArrayList<>();
    Map<String, ConfigKey> configKeys = new HashMap<>();
    List<String> allGroups = new ArrayList<>();
    // do basic connector validation (name, connector type, etc.)
    Map<String, ConfigValue> validatedConnectorConfig = validateBasicConnectorConfig(connector, connectorConfigDef, connectorConfig);
    configValues.addAll(validatedConnectorConfig.values());
    configKeys.putAll(connectorConfigDef.configKeys());
    allGroups.addAll(connectorConfigDef.groups());
    // do custom connector-specific validation
    Config config = connector.validate(connectorConfig);
    ConfigDef configDef = connector.config();
    configKeys.putAll(configDef.configKeys());
    allGroups.addAll(configDef.groups());
    configValues.addAll(config.configValues());
    return generateResult(connType, configKeys, configValues, allGroups);
}

Example 7 with ConfigValue

use of org.apache.kafka.common.config.ConfigValue in project kafka by apache.

the class AbstractHerder method generateResult.

// public for testing
public static ConfigInfos generateResult(String connType, Map<String, ConfigKey> configKeys, List<ConfigValue> configValues, List<String> groups) {
    int errorCount = 0;
    List<ConfigInfo> configInfoList = new LinkedList<>();
    Map<String, ConfigValue> configValueMap = new HashMap<>();
    for (ConfigValue configValue : configValues) {
        String configName = configValue.name();
        configValueMap.put(configName, configValue);
        if (!configKeys.containsKey(configName)) {
            configValue.addErrorMessage("Configuration is not defined: " + configName);
            configInfoList.add(new ConfigInfo(null, convertConfigValue(configValue, null)));
        }
    }
    for (Map.Entry<String, ConfigKey> entry : configKeys.entrySet()) {
        String configName = entry.getKey();
        ConfigKeyInfo configKeyInfo = convertConfigKey(entry.getValue());
        Type type = entry.getValue().type;
        ConfigValueInfo configValueInfo = null;
        if (configValueMap.containsKey(configName)) {
            ConfigValue configValue = configValueMap.get(configName);
            configValueInfo = convertConfigValue(configValue, type);
            errorCount += configValue.errorMessages().size();
        }
        configInfoList.add(new ConfigInfo(configKeyInfo, configValueInfo));
    }
    return new ConfigInfos(connType, errorCount, groups, configInfoList);
}

Example 8 with ConfigValue

use of org.apache.kafka.common.config.ConfigValue in project kafka by apache.

the class StandaloneHerderTest method testCreateConnectorFailedCustomValidation.

@Test
public void testCreateConnectorFailedCustomValidation() throws Exception {
    connector = PowerMock.createMock(BogusSourceConnector.class);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    ConfigDef configDef = new ConfigDef();
    configDef.define("foo.bar", ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, "foo.bar doc");
    EasyMock.expect(connectorMock.config()).andReturn(configDef);
    ConfigValue validatedValue = new ConfigValue("foo.bar");
    validatedValue.addErrorMessage("Failed foo.bar validation");
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(singletonList(validatedValue)));
    createCallback.onCompletion(EasyMock.<BadRequestException>anyObject(), EasyMock.<Herder.Created<ConnectorInfo>>isNull());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    PowerMock.verifyAll();
}

Example 9 with ConfigValue

use of org.apache.kafka.common.config.ConfigValue in project kafka by apache.

the class DistributedHerderTest method testConnectorNameConflictsWithWorkerGroupId.

@Test
public void testConnectorNameConflictsWithWorkerGroupId() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    member.wakeup();
    PowerMock.expectLastCall();
    Map<String, String> config = new HashMap<>(CONN2_CONFIG);
    config.put(ConnectorConfig.NAME_CONFIG, "test-group");
    // config validation
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(SinkConnector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    // CONN2 creation should fail because the worker group id (connect-test-group) conflicts with
    // the consumer group id we would use for this sink
    Capture<Throwable> error = EasyMock.newCapture();
    putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.isNull(Herder.Created.class));
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // No immediate action besides this -- change will be picked up via the config log
    PowerMock.replayAll();
    herder.putConnectorConfig(CONN2, config, false, putConnectorCallback);
    herder.tick();
    assertTrue(error.hasCaptured());
    assertTrue(error.getValue() instanceof BadRequestException);
    PowerMock.verifyAll();
}

Example 10 with ConfigValue

use of org.apache.kafka.common.config.ConfigValue in project kafka by apache.

the class DistributedHerderTest method testCreateConnector.

@Test
public void testCreateConnector() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    member.wakeup();
    PowerMock.expectLastCall();
    // config validation
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(CONN2_CONFIG)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    // CONN2 is new, should succeed
    configBackingStore.putConnectorConfig(CONN2, CONN2_CONFIG);
    PowerMock.expectLastCall();
    ConnectorInfo info = new ConnectorInfo(CONN2, CONN2_CONFIG, Collections.<ConnectorTaskId>emptyList());
    putConnectorCallback.onCompletion(null, new Herder.Created<>(true, info));
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // No immediate action besides this -- change will be picked up via the config log
    PowerMock.replayAll();
    herder.putConnectorConfig(CONN2, CONN2_CONFIG, false, putConnectorCallback);
    herder.tick();
    PowerMock.verifyAll();
}