Search in sources :

Example 31 with ConnectorInfo

use of org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo in project kafka by apache.

the class StandaloneHerderTest method testCreateAndStop.

@Test
public void testCreateAndStop() throws Exception {
    connector = PowerMock.createMock(BogusSourceConnector.class);
    expectAdd(SourceSink.SOURCE);
    Map<String, String> connectorConfig = connectorConfig(SourceSink.SOURCE);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    expectConfigValidation(connectorMock, true, connectorConfig);
    // herder.stop() should stop any running connectors and tasks even if destroyConnector was not invoked
    expectStop();
    statusBackingStore.put(new TaskStatus(new ConnectorTaskId(CONNECTOR_NAME, 0), AbstractStatus.State.DESTROYED, WORKER_ID, 0));
    statusBackingStore.stop();
    EasyMock.expectLastCall();
    worker.stop();
    EasyMock.expectLastCall();
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, connectorConfig, false, createCallback);
    Herder.Created<ConnectorInfo> connectorInfo = createCallback.get(1000L, TimeUnit.SECONDS);
    assertEquals(createdInfo(SourceSink.SOURCE), connectorInfo.result());
    herder.stop();
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) WorkerConnector(org.apache.kafka.connect.runtime.WorkerConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConnectorTaskId(org.apache.kafka.connect.util.ConnectorTaskId) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) TaskStatus(org.apache.kafka.connect.runtime.TaskStatus) Herder(org.apache.kafka.connect.runtime.Herder) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 32 with ConnectorInfo

use of org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo in project kafka by apache.

the class StandaloneHerderTest method testAccessors.

@Test
public void testAccessors() throws Exception {
    Map<String, String> connConfig = connectorConfig(SourceSink.SOURCE);
    System.out.println(connConfig);
    Callback<Collection<String>> listConnectorsCb = PowerMock.createMock(Callback.class);
    Callback<ConnectorInfo> connectorInfoCb = PowerMock.createMock(Callback.class);
    Callback<Map<String, String>> connectorConfigCb = PowerMock.createMock(Callback.class);
    Callback<List<TaskInfo>> taskConfigsCb = PowerMock.createMock(Callback.class);
    // Check accessors with empty worker
    listConnectorsCb.onCompletion(null, Collections.EMPTY_SET);
    EasyMock.expectLastCall();
    connectorInfoCb.onCompletion(EasyMock.<NotFoundException>anyObject(), EasyMock.isNull());
    EasyMock.expectLastCall();
    connectorConfigCb.onCompletion(EasyMock.<NotFoundException>anyObject(), EasyMock.isNull());
    EasyMock.expectLastCall();
    taskConfigsCb.onCompletion(EasyMock.<NotFoundException>anyObject(), EasyMock.isNull());
    EasyMock.expectLastCall();
    // Create connector
    connector = PowerMock.createMock(BogusSourceConnector.class);
    expectAdd(SourceSink.SOURCE);
    expectConfigValidation(connector, true, connConfig);
    // Validate accessors with 1 connector
    listConnectorsCb.onCompletion(null, singleton(CONNECTOR_NAME));
    EasyMock.expectLastCall();
    ConnectorInfo connInfo = new ConnectorInfo(CONNECTOR_NAME, connConfig, Arrays.asList(new ConnectorTaskId(CONNECTOR_NAME, 0)), ConnectorType.SOURCE);
    connectorInfoCb.onCompletion(null, connInfo);
    EasyMock.expectLastCall();
    connectorConfigCb.onCompletion(null, connConfig);
    EasyMock.expectLastCall();
    TaskInfo taskInfo = new TaskInfo(new ConnectorTaskId(CONNECTOR_NAME, 0), taskConfig(SourceSink.SOURCE));
    taskConfigsCb.onCompletion(null, Arrays.asList(taskInfo));
    EasyMock.expectLastCall();
    PowerMock.replayAll();
    // All operations are synchronous for StandaloneHerder, so we don't need to actually wait after making each call
    herder.connectors(listConnectorsCb);
    herder.connectorInfo(CONNECTOR_NAME, connectorInfoCb);
    herder.connectorConfig(CONNECTOR_NAME, connectorConfigCb);
    herder.taskConfigs(CONNECTOR_NAME, taskConfigsCb);
    herder.putConnectorConfig(CONNECTOR_NAME, connConfig, false, createCallback);
    Herder.Created<ConnectorInfo> connectorInfo = createCallback.get(1000L, TimeUnit.SECONDS);
    assertEquals(createdInfo(SourceSink.SOURCE), connectorInfo.result());
    EasyMock.reset(transformer);
    EasyMock.expect(transformer.transform(EasyMock.eq(CONNECTOR_NAME), EasyMock.anyObject())).andThrow(new AssertionError("Config transformation should not occur when requesting connector or task info")).anyTimes();
    EasyMock.replay(transformer);
    herder.connectors(listConnectorsCb);
    herder.connectorInfo(CONNECTOR_NAME, connectorInfoCb);
    herder.connectorConfig(CONNECTOR_NAME, connectorConfigCb);
    herder.taskConfigs(CONNECTOR_NAME, taskConfigsCb);
    PowerMock.verifyAll();
}
Also used : ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) ConnectorTaskId(org.apache.kafka.connect.util.ConnectorTaskId) TaskInfo(org.apache.kafka.connect.runtime.rest.entities.TaskInfo) Collection(java.util.Collection) Collections.singletonList(java.util.Collections.singletonList) List(java.util.List) ArrayList(java.util.ArrayList) Map(java.util.Map) HashMap(java.util.HashMap) Collections.singletonMap(java.util.Collections.singletonMap) Herder(org.apache.kafka.connect.runtime.Herder) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 33 with ConnectorInfo

use of org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo in project kafka by apache.

the class StandaloneHerderTest method testCreateSourceConnector.

@Test
public void testCreateSourceConnector() throws Exception {
    connector = PowerMock.createMock(BogusSourceConnector.class);
    expectAdd(SourceSink.SOURCE);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    expectConfigValidation(connectorMock, true, config);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    Herder.Created<ConnectorInfo> connectorInfo = createCallback.get(1000L, TimeUnit.SECONDS);
    assertEquals(createdInfo(SourceSink.SOURCE), connectorInfo.result());
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) WorkerConnector(org.apache.kafka.connect.runtime.WorkerConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) Herder(org.apache.kafka.connect.runtime.Herder) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 34 with ConnectorInfo

use of org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo in project kafka by apache.

the class StandaloneHerderTest method testRestartTask.

@Test
public void testRestartTask() throws Exception {
    ConnectorTaskId taskId = new ConnectorTaskId(CONNECTOR_NAME, 0);
    expectAdd(SourceSink.SOURCE);
    Map<String, String> connectorConfig = connectorConfig(SourceSink.SOURCE);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    expectConfigValidation(connectorMock, true, connectorConfig);
    worker.stopAndAwaitTask(taskId);
    EasyMock.expectLastCall();
    ClusterConfigState configState = new ClusterConfigState(-1, null, Collections.singletonMap(CONNECTOR_NAME, 1), Collections.singletonMap(CONNECTOR_NAME, connectorConfig), Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED), Collections.singletonMap(taskId, taskConfig(SourceSink.SOURCE)), new HashSet<>(), transformer);
    worker.startTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SOURCE), herder, TargetState.STARTED);
    EasyMock.expectLastCall().andReturn(true);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, connectorConfig, false, createCallback);
    Herder.Created<ConnectorInfo> connectorInfo = createCallback.get(1000L, TimeUnit.SECONDS);
    createCallback.get(1000L, TimeUnit.SECONDS);
    assertEquals(createdInfo(SourceSink.SOURCE), connectorInfo.result());
    FutureCallback<Void> restartTaskCallback = new FutureCallback<>();
    herder.restartTask(taskId, restartTaskCallback);
    restartTaskCallback.get(1000L, TimeUnit.MILLISECONDS);
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) WorkerConnector(org.apache.kafka.connect.runtime.WorkerConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConnectorTaskId(org.apache.kafka.connect.util.ConnectorTaskId) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) Herder(org.apache.kafka.connect.runtime.Herder) ClusterConfigState(org.apache.kafka.connect.runtime.distributed.ClusterConfigState) FutureCallback(org.apache.kafka.connect.util.FutureCallback) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 35 with ConnectorInfo

use of org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo in project kafka by apache.

the class StandaloneHerderTest method testRestartTaskFailureOnStart.

@Test
public void testRestartTaskFailureOnStart() throws Exception {
    ConnectorTaskId taskId = new ConnectorTaskId(CONNECTOR_NAME, 0);
    expectAdd(SourceSink.SOURCE);
    Map<String, String> connectorConfig = connectorConfig(SourceSink.SOURCE);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    expectConfigValidation(connectorMock, true, connectorConfig);
    worker.stopAndAwaitTask(taskId);
    EasyMock.expectLastCall();
    ClusterConfigState configState = new ClusterConfigState(-1, null, Collections.singletonMap(CONNECTOR_NAME, 1), Collections.singletonMap(CONNECTOR_NAME, connectorConfig), Collections.singletonMap(CONNECTOR_NAME, TargetState.STARTED), Collections.singletonMap(new ConnectorTaskId(CONNECTOR_NAME, 0), taskConfig(SourceSink.SOURCE)), new HashSet<>(), transformer);
    worker.startTask(taskId, configState, connectorConfig, taskConfig(SourceSink.SOURCE), herder, TargetState.STARTED);
    EasyMock.expectLastCall().andReturn(false);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, connectorConfig, false, createCallback);
    Herder.Created<ConnectorInfo> connectorInfo = createCallback.get(1000L, TimeUnit.MILLISECONDS);
    assertEquals(createdInfo(SourceSink.SOURCE), connectorInfo.result());
    FutureCallback<Void> cb = new FutureCallback<>();
    herder.restartTask(taskId, cb);
    try {
        cb.get(1000L, TimeUnit.MILLISECONDS);
        fail("Expected restart callback to raise an exception");
    } catch (ExecutionException exception) {
        assertEquals(ConnectException.class, exception.getCause().getClass());
    }
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) WorkerConnector(org.apache.kafka.connect.runtime.WorkerConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConnectorTaskId(org.apache.kafka.connect.util.ConnectorTaskId) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) ExecutionException(java.util.concurrent.ExecutionException) Herder(org.apache.kafka.connect.runtime.Herder) ClusterConfigState(org.apache.kafka.connect.runtime.distributed.ClusterConfigState) FutureCallback(org.apache.kafka.connect.util.FutureCallback) ConnectException(org.apache.kafka.connect.errors.ConnectException) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

ConnectorInfo (org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo)78 Herder (org.apache.kafka.connect.runtime.Herder)65 Test (org.junit.Test)63 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)61 Callback (org.apache.kafka.connect.util.Callback)35 Connector (org.apache.kafka.connect.connector.Connector)28 SinkConnector (org.apache.kafka.connect.sink.SinkConnector)28 SourceConnector (org.apache.kafka.connect.source.SourceConnector)28 FutureCallback (org.apache.kafka.connect.util.FutureCallback)25 WorkerConnector (org.apache.kafka.connect.runtime.WorkerConnector)20 CreateConnectorRequest (org.apache.kafka.connect.runtime.rest.entities.CreateConnectorRequest)19 HashMap (java.util.HashMap)17 URI (java.net.URI)13 Map (java.util.Map)13 ConnectorConfig (org.apache.kafka.connect.runtime.ConnectorConfig)11 ConnectorTaskId (org.apache.kafka.connect.util.ConnectorTaskId)11 Config (org.apache.kafka.common.config.Config)10 ConfigDef (org.apache.kafka.common.config.ConfigDef)10 ConfigValue (org.apache.kafka.common.config.ConfigValue)10 TaskConfig (org.apache.kafka.connect.runtime.TaskConfig)10