Search in sources :

Example 6 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class DistributedHerderTest method testPutConnectorConfig.

@Test
public void testPutConnectorConfig() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    expectRebalance(1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
    PowerMock.expectLastCall().andReturn(true);
    EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
    EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS);
    // list connectors, get connector info, get connector config, get task configs
    member.wakeup();
    PowerMock.expectLastCall().anyTimes();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // Poll loop for second round of calls
    member.ensureActive();
    PowerMock.expectLastCall();
    // config validation
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(CONN1_CONFIG_UPDATED)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    configBackingStore.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED);
    PowerMock.expectLastCall().andAnswer(new IAnswer<Object>() {

        @Override
        public Object answer() throws Throwable {
            // Simulate response to writing config + waiting until end of log to be read
            configUpdateListener.onConnectorConfigUpdate(CONN1);
            return null;
        }
    });
    // As a result of reconfig, should need to update snapshot. With only connector updates, we'll just restart
    // connector without rebalance
    EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT_UPDATED_CONN1_CONFIG);
    worker.stopConnector(CONN1);
    PowerMock.expectLastCall().andReturn(true);
    worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
    PowerMock.expectLastCall().andReturn(true);
    EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
    EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS);
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // Third tick just to read the config
    member.ensureActive();
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    // Should pick up original config
    FutureCallback<Map<String, String>> connectorConfigCb = new FutureCallback<>();
    herder.connectorConfig(CONN1, connectorConfigCb);
    herder.tick();
    assertTrue(connectorConfigCb.isDone());
    assertEquals(CONN1_CONFIG, connectorConfigCb.get());
    // Apply new config.
    FutureCallback<Herder.Created<ConnectorInfo>> putConfigCb = new FutureCallback<>();
    herder.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED, true, putConfigCb);
    herder.tick();
    assertTrue(putConfigCb.isDone());
    ConnectorInfo updatedInfo = new ConnectorInfo(CONN1, CONN1_CONFIG_UPDATED, Arrays.asList(TASK0, TASK1, TASK2));
    assertEquals(new Herder.Created<>(false, updatedInfo), putConfigCb.get());
    // Check config again to validate change
    connectorConfigCb = new FutureCallback<>();
    herder.connectorConfig(CONN1, connectorConfigCb);
    herder.tick();
    assertTrue(connectorConfigCb.isDone());
    assertEquals(CONN1_CONFIG_UPDATED, connectorConfigCb.get());
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConfigValue(org.apache.kafka.common.config.ConfigValue) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) WorkerConfig(org.apache.kafka.connect.runtime.WorkerConfig) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) Config(org.apache.kafka.common.config.Config) SinkConnectorConfig(org.apache.kafka.connect.runtime.SinkConnectorConfig) TaskConfig(org.apache.kafka.connect.runtime.TaskConfig) ConnectorFactory(org.apache.kafka.connect.runtime.ConnectorFactory) ConfigDef(org.apache.kafka.common.config.ConfigDef) Map(java.util.Map) HashMap(java.util.HashMap) Herder(org.apache.kafka.connect.runtime.Herder) FutureCallback(org.apache.kafka.connect.util.FutureCallback) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 7 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class DistributedHerderTest method testCreateConnectorFailedCustomValidation.

@Test
public void testCreateConnectorFailedCustomValidation() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    member.wakeup();
    PowerMock.expectLastCall();
    // config validation
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    ConfigDef configDef = new ConfigDef();
    configDef.define("foo.bar", ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, "foo.bar doc");
    EasyMock.expect(connectorMock.config()).andReturn(configDef);
    ConfigValue validatedValue = new ConfigValue("foo.bar");
    validatedValue.addErrorMessage("Failed foo.bar validation");
    EasyMock.expect(connectorMock.validate(CONN2_CONFIG)).andReturn(new Config(singletonList(validatedValue)));
    // CONN2 creation should fail
    Capture<Throwable> error = EasyMock.newCapture();
    putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.<Herder.Created<ConnectorInfo>>isNull());
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // No immediate action besides this -- change will be picked up via the config log
    PowerMock.replayAll();
    herder.putConnectorConfig(CONN2, CONN2_CONFIG, false, putConnectorCallback);
    herder.tick();
    assertTrue(error.hasCaptured());
    assertTrue(error.getValue() instanceof BadRequestException);
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConfigValue(org.apache.kafka.common.config.ConfigValue) ConnectorFactory(org.apache.kafka.connect.runtime.ConnectorFactory) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) WorkerConfig(org.apache.kafka.connect.runtime.WorkerConfig) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) Config(org.apache.kafka.common.config.Config) SinkConnectorConfig(org.apache.kafka.connect.runtime.SinkConnectorConfig) TaskConfig(org.apache.kafka.connect.runtime.TaskConfig) BadRequestException(org.apache.kafka.connect.runtime.rest.errors.BadRequestException) ConfigDef(org.apache.kafka.common.config.ConfigDef) Herder(org.apache.kafka.connect.runtime.Herder) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 8 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class DistributedHerderTest method testCreateConnectorFailedBasicValidation.

@Test
public void testCreateConnectorFailedBasicValidation() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    HashMap<String, String> config = new HashMap<>(CONN2_CONFIG);
    config.remove(ConnectorConfig.NAME_CONFIG);
    member.wakeup();
    PowerMock.expectLastCall();
    // config validation
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    // CONN2 creation should fail
    Capture<Throwable> error = EasyMock.newCapture();
    putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.<Herder.Created<ConnectorInfo>>isNull());
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // No immediate action besides this -- change will be picked up via the config log
    PowerMock.replayAll();
    herder.putConnectorConfig(CONN2, config, false, putConnectorCallback);
    herder.tick();
    assertTrue(error.hasCaptured());
    assertTrue(error.getValue() instanceof BadRequestException);
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConfigValue(org.apache.kafka.common.config.ConfigValue) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) HashMap(java.util.HashMap) WorkerConfig(org.apache.kafka.connect.runtime.WorkerConfig) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) Config(org.apache.kafka.common.config.Config) SinkConnectorConfig(org.apache.kafka.connect.runtime.SinkConnectorConfig) TaskConfig(org.apache.kafka.connect.runtime.TaskConfig) ConnectorFactory(org.apache.kafka.connect.runtime.ConnectorFactory) BadRequestException(org.apache.kafka.connect.runtime.rest.errors.BadRequestException) ConfigDef(org.apache.kafka.common.config.ConfigDef) Herder(org.apache.kafka.connect.runtime.Herder) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 9 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class StandaloneHerderTest method testCreateConnectorFailedBasicValidation.

@Test
public void testCreateConnectorFailedBasicValidation() throws Exception {
    connector = PowerMock.createMock(BogusSourceConnector.class);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    config.remove(ConnectorConfig.NAME_CONFIG);
    ConnectorFactory connectorFactoryMock = PowerMock.createMock(ConnectorFactory.class);
    EasyMock.expect(worker.getConnectorFactory()).andStubReturn(connectorFactoryMock);
    Connector connectorMock = PowerMock.createMock(Connector.class);
    EasyMock.expect(connectorFactoryMock.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andStubReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    createCallback.onCompletion(EasyMock.<BadRequestException>anyObject(), EasyMock.<Herder.Created<ConnectorInfo>>isNull());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConfigValue(org.apache.kafka.common.config.ConfigValue) ConnectorFactory(org.apache.kafka.connect.runtime.ConnectorFactory) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) Config(org.apache.kafka.common.config.Config) TaskConfig(org.apache.kafka.connect.runtime.TaskConfig) ConfigDef(org.apache.kafka.common.config.ConfigDef) Herder(org.apache.kafka.connect.runtime.Herder) Test(org.junit.Test)

Example 10 with Connector

use of org.apache.kafka.connect.connector.Connector in project kafka by apache.

the class Worker method startConnector.

/**
     * Start a connector managed by this worker.
     *
     * @param connName the connector name.
     * @param connProps the properties of the connector.
     * @param ctx the connector runtime context.
     * @param statusListener a listener for the runtime status transitions of the connector.
     * @param initialState the initial state of the connector.
     * @return true if the connector started successfully.
     */
public boolean startConnector(String connName, Map<String, String> connProps, ConnectorContext ctx, ConnectorStatus.Listener statusListener, TargetState initialState) {
    if (connectors.containsKey(connName))
        throw new ConnectException("Connector with name " + connName + " already exists");
    final WorkerConnector workerConnector;
    try {
        final ConnectorConfig connConfig = new ConnectorConfig(connProps);
        final String connClass = connConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG);
        log.info("Creating connector {} of type {}", connName, connClass);
        final Connector connector = connectorFactory.newConnector(connClass);
        workerConnector = new WorkerConnector(connName, connector, ctx, statusListener);
        log.info("Instantiated connector {} with version {} of type {}", connName, connector.version(), connector.getClass());
        workerConnector.initialize(connConfig);
        workerConnector.transitionTo(initialState);
    } catch (Throwable t) {
        log.error("Failed to start connector {}", connName, t);
        statusListener.onFailure(connName, t);
        return false;
    }
    WorkerConnector existing = connectors.putIfAbsent(connName, workerConnector);
    if (existing != null)
        throw new ConnectException("Connector with name " + connName + " already exists");
    log.info("Finished creating connector {}", connName);
    return true;
}
Also used : Connector(org.apache.kafka.connect.connector.Connector) ConnectException(org.apache.kafka.connect.errors.ConnectException)

Aggregations

Connector (org.apache.kafka.connect.connector.Connector)19 Test (org.junit.Test)12 SourceConnector (org.apache.kafka.connect.source.SourceConnector)11 HashMap (java.util.HashMap)10 Config (org.apache.kafka.common.config.Config)10 ConfigDef (org.apache.kafka.common.config.ConfigDef)10 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)10 ConnectorConfig (org.apache.kafka.connect.runtime.ConnectorConfig)9 SinkConnector (org.apache.kafka.connect.sink.SinkConnector)9 ConfigValue (org.apache.kafka.common.config.ConfigValue)8 ConnectorFactory (org.apache.kafka.connect.runtime.ConnectorFactory)8 TaskConfig (org.apache.kafka.connect.runtime.TaskConfig)8 Herder (org.apache.kafka.connect.runtime.Herder)6 ConnectorInfo (org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo)6 ConnectException (org.apache.kafka.connect.errors.ConnectException)5 SinkConnectorConfig (org.apache.kafka.connect.runtime.SinkConnectorConfig)5 WorkerConfig (org.apache.kafka.connect.runtime.WorkerConfig)5 ArrayList (java.util.ArrayList)4 Map (java.util.Map)4 ConnectorContext (org.apache.kafka.connect.connector.ConnectorContext)4