Examples with ConnectorConfig org.apache.kafka.connect.runtime.ConnectorConfig used on opensource projects

Example 1 with ConnectorConfig

use of org.apache.kafka.connect.runtime.ConnectorConfig in project kafka by apache.

the class StandaloneHerder method recomputeTaskConfigs.

private List<Map<String, String>> recomputeTaskConfigs(String connName) {
    Map<String, String> config = configState.connectorConfig(connName);
    ConnectorConfig connConfig;
    if (worker.isSinkConnector(connName)) {
        connConfig = new SinkConnectorConfig(config);
        return worker.connectorTaskConfigs(connName, connConfig.getInt(ConnectorConfig.TASKS_MAX_CONFIG), connConfig.getList(SinkConnectorConfig.TOPICS_CONFIG));
    } else {
        connConfig = new SourceConnectorConfig(config);
        return worker.connectorTaskConfigs(connName, connConfig.getInt(ConnectorConfig.TASKS_MAX_CONFIG), null);
    }
}

Example 2 with ConnectorConfig

use of org.apache.kafka.connect.runtime.ConnectorConfig in project kafka by apache.

the class StandaloneHerderTest method testCreateConnectorFailedBasicValidation.

@Test
public void testCreateConnectorFailedBasicValidation() throws Exception {
    connector = PowerMock.createMock(BogusSourceConnector.class);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    config.remove(ConnectorConfig.NAME_CONFIG);
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andStubReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    createCallback.onCompletion(EasyMock.<BadRequestException>anyObject(), EasyMock.<Herder.Created<ConnectorInfo>>isNull());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    PowerMock.verifyAll();
}

Example 3 with ConnectorConfig

use of org.apache.kafka.connect.runtime.ConnectorConfig in project kafka by apache.

the class DistributedHerder method reconfigureConnector.

// Updates configurations for a connector by requesting them from the connector, filling in parameters provided
// by the system, then checks whether any configs have actually changed before submitting the new configs to storage
private void reconfigureConnector(final String connName, final Callback<Void> cb) {
    try {
        if (!worker.isRunning(connName)) {
            log.info("Skipping reconfiguration of connector {} since it is not running", connName);
            return;
        }
        Map<String, String> configs = configState.connectorConfig(connName);
        ConnectorConfig connConfig;
        List<String> sinkTopics = null;
        if (worker.isSinkConnector(connName)) {
            connConfig = new SinkConnectorConfig(configs);
            sinkTopics = connConfig.getList(SinkConnectorConfig.TOPICS_CONFIG);
        } else {
            connConfig = new SourceConnectorConfig(configs);
        }
        final List<Map<String, String>> taskProps = worker.connectorTaskConfigs(connName, connConfig.getInt(ConnectorConfig.TASKS_MAX_CONFIG), sinkTopics);
        boolean changed = false;
        int currentNumTasks = configState.taskCount(connName);
        if (taskProps.size() != currentNumTasks) {
            log.debug("Change in connector task count from {} to {}, writing updated task configurations", currentNumTasks, taskProps.size());
            changed = true;
        } else {
            int index = 0;
            for (Map<String, String> taskConfig : taskProps) {
                if (!taskConfig.equals(configState.taskConfig(new ConnectorTaskId(connName, index)))) {
                    log.debug("Change in task configurations, writing updated task configurations");
                    changed = true;
                    break;
                }
                index++;
            }
        }
        if (changed) {
            if (isLeader()) {
                configBackingStore.putTaskConfigs(connName, taskProps);
                cb.onCompletion(null, null);
            } else {
                // We cannot forward the request on the same thread because this reconfiguration can happen as a result of connector
                // addition or removal. If we blocked waiting for the response from leader, we may be kicked out of the worker group.
                forwardRequestExecutor.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            String reconfigUrl = RestServer.urlJoin(leaderUrl(), "/connectors/" + connName + "/tasks");
                            RestServer.httpRequest(reconfigUrl, "POST", taskProps, null);
                            cb.onCompletion(null, null);
                        } catch (ConnectException e) {
                            log.error("Request to leader to reconfigure connector tasks failed", e);
                            cb.onCompletion(e, null);
                        }
                    }
                });
            }
        }
    } catch (Throwable t) {
        cb.onCompletion(t, null);
    }
}

Example 4 with ConnectorConfig

use of org.apache.kafka.connect.runtime.ConnectorConfig in project kafka by apache.

the class StandaloneHerderTest method testCreateConnectorFailedCustomValidation.

@Test
public void testCreateConnectorFailedCustomValidation() throws Exception {
    connector = PowerMock.createMock(BogusSourceConnector.class);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    ConfigDef configDef = new ConfigDef();
    configDef.define("foo.bar", ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, "foo.bar doc");
    EasyMock.expect(connectorMock.config()).andReturn(configDef);
    ConfigValue validatedValue = new ConfigValue("foo.bar");
    validatedValue.addErrorMessage("Failed foo.bar validation");
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(singletonList(validatedValue)));
    createCallback.onCompletion(EasyMock.<BadRequestException>anyObject(), EasyMock.<Herder.Created<ConnectorInfo>>isNull());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    PowerMock.verifyAll();
}