Examples with ConfigDef org.apache.kafka.common.config.ConfigDef used on opensource projects

Example 46 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project connect-utils by jcustenborder.

the class MarkdownFormatterTest method toMarkdownConfigDef.

@Test
public void toMarkdownConfigDef() {
    final String expected = "| Name                  | Description                                                                                                                                                                                                                                                                                    | Type     | Default               | Valid Values | Importance |\n" + "|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-----------------------|--------------|------------|\n" + "| ssl.key.password      | The password of the private key in the key store file. This is optional for client.                                                                                                                                                                                                            | password | null                  |              | high       |\n" + "| ssl.keystore.location | The location of the key store file. This is optional for client and can be used for two-way authentication for client.                                                                                                                                                                         | string   | null                  |              | high       |\n" + "| ssl.enabled.protocols | The SSL protocol used to generate the SSLContext. Default setting is TLS, which is fine for most cases. Allowed values in recent JVMs are TLS, TLSv1.1 and TLSv1.2. SSL, SSLv2 and SSLv3 may be supported in older JVMs, but their usage is discouraged due to known security vulnerabilities. | string   | TLSv1.2,TLSv1.1,TLSv1 |              | medium     |\n";
    ConfigDef def = new ConfigDef().define("ssl.key.password", ConfigDef.Type.PASSWORD, null, ConfigDef.Importance.HIGH, "The password of the private key in the key store file. This is optional for client.").define("ssl.keystore.location", ConfigDef.Type.STRING, null, ConfigDef.Importance.HIGH, "The location of the key store file. This is optional for client and can be used for two-way authentication for client.").define("ssl.enabled.protocols", ConfigDef.Type.STRING, "TLSv1.2,TLSv1.1,TLSv1", ConfigDef.Importance.MEDIUM, "The SSL protocol used to generate the SSLContext. Default setting is TLS, which is fine for most cases. Allowed values in recent JVMs are TLS, TLSv1.1 and TLSv1.2. SSL, SSLv2 and SSLv3 may be supported in older JVMs, but their usage is discouraged due to known security vulnerabilities.");
    final String actual = MarkdownFormatter.toMarkdown(def);
    log.trace("\n{}", actual);
    assertEquals(expected, actual);
}

Example 47 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project apache-kafka-on-k8s by banzaicloud.

the class AbstractHerder method validateConnectorConfig.

@Override
public ConfigInfos validateConnectorConfig(Map<String, String> connectorProps) {
    String connType = connectorProps.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
    if (connType == null)
        throw new BadRequestException("Connector config " + connectorProps + " contains no connector type");
    Connector connector = getConnector(connType);
    ClassLoader savedLoader = plugins().compareAndSwapLoaders(connector);
    try {
        ConfigDef baseConfigDef;
        if (connector instanceof SourceConnector) {
            baseConfigDef = SourceConnectorConfig.configDef();
        } else {
            baseConfigDef = SinkConnectorConfig.configDef();
            SinkConnectorConfig.validate(connectorProps);
        }
        ConfigDef enrichedConfigDef = ConnectorConfig.enrich(plugins(), baseConfigDef, connectorProps, false);
        Map<String, ConfigValue> validatedConnectorConfig = validateBasicConnectorConfig(connector, enrichedConfigDef, connectorProps);
        List<ConfigValue> configValues = new ArrayList<>(validatedConnectorConfig.values());
        Map<String, ConfigKey> configKeys = new LinkedHashMap<>(enrichedConfigDef.configKeys());
        Set<String> allGroups = new LinkedHashSet<>(enrichedConfigDef.groups());
        // do custom connector-specific validation
        Config config = connector.validate(connectorProps);
        ConfigDef configDef = connector.config();
        configKeys.putAll(configDef.configKeys());
        allGroups.addAll(configDef.groups());
        configValues.addAll(config.configValues());
        return generateResult(connType, configKeys, configValues, new ArrayList<>(allGroups));
    } finally {
        Plugins.compareAndSwapLoaders(savedLoader);
    }
}

Example 48 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project apache-kafka-on-k8s by banzaicloud.

the class Connector method validate.

/**
 * Validate the connector configuration values against configuration definitions.
 * @param connectorConfigs the provided configuration values
 * @return List of Config, each Config contains the updated configuration information given
 * the current configuration values.
 */
public Config validate(Map<String, String> connectorConfigs) {
    ConfigDef configDef = config();
    List<ConfigValue> configValues = configDef.validate(connectorConfigs);
    return new Config(configValues);
}

Example 49 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project apache-kafka-on-k8s by banzaicloud.

the class DistributedHerderTest method testConnectorNameConflictsWithWorkerGroupId.

@Test
public void testConnectorNameConflictsWithWorkerGroupId() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    member.wakeup();
    PowerMock.expectLastCall();
    Map<String, String> config = new HashMap<>(CONN2_CONFIG);
    config.put(ConnectorConfig.NAME_CONFIG, "test-group");
    // config validation
    Connector connectorMock = PowerMock.createMock(SinkConnector.class);
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
    EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
    EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
    // CONN2 creation should fail because the worker group id (connect-test-group) conflicts with
    // the consumer group id we would use for this sink
    Capture<Throwable> error = EasyMock.newCapture();
    putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.isNull(Herder.Created.class));
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // No immediate action besides this -- change will be picked up via the config log
    PowerMock.replayAll();
    herder.putConnectorConfig(CONN2, config, false, putConnectorCallback);
    herder.tick();
    assertTrue(error.hasCaptured());
    assertTrue(error.getValue() instanceof BadRequestException);
    time.sleep(1000L);
    assertStatistics(3, 1, 100, 1000L);
    PowerMock.verifyAll();
}

Example 50 with ConfigDef

use of org.apache.kafka.common.config.ConfigDef in project apache-kafka-on-k8s by banzaicloud.

the class DistributedHerderTest method testPutConnectorConfig.

@Test
public void testPutConnectorConfig() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    expectRebalance(1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
    PowerMock.expectLastCall().andReturn(true);
    EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
    EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS);
    // list connectors, get connector info, get connector config, get task configs
    member.wakeup();
    PowerMock.expectLastCall().anyTimes();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // Poll loop for second round of calls
    member.ensureActive();
    PowerMock.expectLastCall();
    // config validation
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes();
    EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
    EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(CONN1_CONFIG_UPDATED)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
    configBackingStore.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED);
    PowerMock.expectLastCall().andAnswer(new IAnswer<Object>() {

        @Override
        public Object answer() throws Throwable {
            // Simulate response to writing config + waiting until end of log to be read
            configUpdateListener.onConnectorConfigUpdate(CONN1);
            return null;
        }
    });
    // As a result of reconfig, should need to update snapshot. With only connector updates, we'll just restart
    // connector without rebalance
    EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT_UPDATED_CONN1_CONFIG);
    worker.stopConnector(CONN1);
    PowerMock.expectLastCall().andReturn(true);
    worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
    PowerMock.expectLastCall().andReturn(true);
    EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
    EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfigUpdated)).andReturn(TASK_CONFIGS);
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // Third tick just to read the config
    member.ensureActive();
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    // Should pick up original config
    FutureCallback<Map<String, String>> connectorConfigCb = new FutureCallback<>();
    herder.connectorConfig(CONN1, connectorConfigCb);
    herder.tick();
    assertTrue(connectorConfigCb.isDone());
    assertEquals(CONN1_CONFIG, connectorConfigCb.get());
    // Apply new config.
    FutureCallback<Herder.Created<ConnectorInfo>> putConfigCb = new FutureCallback<>();
    herder.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED, true, putConfigCb);
    herder.tick();
    assertTrue(putConfigCb.isDone());
    ConnectorInfo updatedInfo = new ConnectorInfo(CONN1, CONN1_CONFIG_UPDATED, Arrays.asList(TASK0, TASK1, TASK2), ConnectorType.SOURCE);
    assertEquals(new Herder.Created<>(false, updatedInfo), putConfigCb.get());
    // Check config again to validate change
    connectorConfigCb = new FutureCallback<>();
    herder.connectorConfig(CONN1, connectorConfigCb);
    herder.tick();
    assertTrue(connectorConfigCb.isDone());
    assertEquals(CONN1_CONFIG_UPDATED, connectorConfigCb.get());
    PowerMock.verifyAll();
}