Examples with Connector org.apache.kafka.connect.connector.Connector used on opensource projects

Example 36 with Connector

use of org.apache.kafka.connect.connector.Connector in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerderTest method testCreateAndStop.

@Test
public void testCreateAndStop() throws Exception {
    connector = PowerMock.createMock(BogusSourceConnector.class);
    expectAdd(SourceSink.SOURCE);
    Map<String, String> connectorConfig = connectorConfig(SourceSink.SOURCE);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    expectConfigValidation(connectorMock, true, connectorConfig);
    // herder.stop() should stop any running connectors and tasks even if destroyConnector was not invoked
    expectStop();
    statusBackingStore.stop();
    EasyMock.expectLastCall();
    worker.stop();
    EasyMock.expectLastCall();
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, connectorConfig, false, createCallback);
    herder.stop();
    PowerMock.verifyAll();
}

Example 37 with Connector

use of org.apache.kafka.connect.connector.Connector in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerderTest method testRestartConnectorFailureOnStart.

@Test
public void testRestartConnectorFailureOnStart() throws Exception {
    expectAdd(SourceSink.SOURCE);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    expectConfigValidation(connectorMock, true, config);
    worker.stopConnector(CONNECTOR_NAME);
    EasyMock.expectLastCall().andReturn(true);
    worker.startConnector(EasyMock.eq(CONNECTOR_NAME), EasyMock.eq(config), EasyMock.anyObject(HerderConnectorContext.class), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
    EasyMock.expectLastCall().andReturn(false);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    FutureCallback<Void> cb = new FutureCallback<>();
    herder.restartConnector(CONNECTOR_NAME, cb);
    try {
        cb.get(1000L, TimeUnit.MILLISECONDS);
        fail();
    } catch (ExecutionException exception) {
        assertEquals(ConnectException.class, exception.getCause().getClass());
    }
    PowerMock.verifyAll();
}

Example 38 with Connector

use of org.apache.kafka.connect.connector.Connector in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerderTest method testCorruptConfig.

@Test
public void testCorruptConfig() {
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.NAME_CONFIG, CONNECTOR_NAME);
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, BogusSinkConnector.class.getName());
    config.put(SinkConnectorConfig.TOPICS_CONFIG, TOPICS_LIST_STR);
    Connector connectorMock = PowerMock.createMock(SinkConnector.class);
    String error = "This is an error in your config!";
    List<String> errors = new ArrayList<>(singletonList(error));
    String key = "foo.invalid.key";
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Arrays.asList(new ConfigValue(key, null, Collections.emptyList(), errors))));
    ConfigDef configDef = new ConfigDef();
    configDef.define(key, ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, "");
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
    EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
    EasyMock.expect(worker.getPlugins()).andStubReturn(plugins);
    EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andStubReturn(configDef);
    EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
    Callback<Herder.Created<ConnectorInfo>> callback = PowerMock.createMock(Callback.class);
    Capture<BadRequestException> capture = Capture.newInstance();
    callback.onCompletion(EasyMock.capture(capture), EasyMock.isNull(Herder.Created.class));
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, true, callback);
    assertEquals(capture.getValue().getMessage(), "Connector configuration is invalid and contains the following 1 error(s):\n" + error + "\n" + "You can also find the above list of errors at the endpoint `/{connectorType}/config/validate`");
    PowerMock.verifyAll();
}

Example 39 with Connector

use of org.apache.kafka.connect.connector.Connector in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerderTest method testCreateConnectorFailedBasicValidation.

@Test
public void testCreateConnectorFailedBasicValidation() throws Exception {
    // Basic validation should be performed and return an error, but should still evaluate the connector's config
    connector = PowerMock.createMock(BogusSourceConnector.class);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    config.remove(ConnectorConfig.NAME_CONFIG);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3);
    EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
    EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andStubReturn(new ConfigDef());
    ConfigValue validatedValue = new ConfigValue("foo.bar");
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(singletonList(validatedValue)));
    EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
    createCallback.onCompletion(EasyMock.<BadRequestException>anyObject(), EasyMock.<Herder.Created<ConnectorInfo>>isNull());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    PowerMock.verifyAll();
}

Example 40 with Connector

use of org.apache.kafka.connect.connector.Connector in project apache-kafka-on-k8s by banzaicloud.

the class Worker method connectorTaskConfigs.

/**
 * Get a list of updated task properties for the tasks of this connector.
 *
 * @param connName the connector name.
 * @return a list of updated tasks properties.
 */
public List<Map<String, String>> connectorTaskConfigs(String connName, ConnectorConfig connConfig) {
    log.trace("Reconfiguring connector tasks for {}", connName);
    WorkerConnector workerConnector = connectors.get(connName);
    if (workerConnector == null)
        throw new ConnectException("Connector " + connName + " not found in this worker.");
    int maxTasks = connConfig.getInt(ConnectorConfig.TASKS_MAX_CONFIG);
    Map<String, String> connOriginals = connConfig.originalsStrings();
    Connector connector = workerConnector.connector();
    List<Map<String, String>> result = new ArrayList<>();
    ClassLoader savedLoader = plugins.currentThreadLoader();
    try {
        savedLoader = plugins.compareAndSwapLoaders(connector);
        String taskClassName = connector.taskClass().getName();
        for (Map<String, String> taskProps : connector.taskConfigs(maxTasks)) {
            // Ensure we don't modify the connector's copy of the config
            Map<String, String> taskConfig = new HashMap<>(taskProps);
            taskConfig.put(TaskConfig.TASK_CLASS_CONFIG, taskClassName);
            if (connOriginals.containsKey(SinkTask.TOPICS_CONFIG)) {
                taskConfig.put(SinkTask.TOPICS_CONFIG, connOriginals.get(SinkTask.TOPICS_CONFIG));
            }
            if (connOriginals.containsKey(SinkTask.TOPICS_REGEX_CONFIG)) {
                taskConfig.put(SinkTask.TOPICS_REGEX_CONFIG, connOriginals.get(SinkTask.TOPICS_REGEX_CONFIG));
            }
            result.add(taskConfig);
        }
    } finally {
        Plugins.compareAndSwapLoaders(savedLoader);
    }
    return result;
}