Search in sources :

Example 31 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project apache-kafka-on-k8s by banzaicloud.

the class DistributedHerderTest method testPutConnectorConfig.

@Test
public void testPutConnectorConfig() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    expectRebalance(1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
    PowerMock.expectLastCall().andReturn(true);
    EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
    EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS);
    // list connectors, get connector info, get connector config, get task configs
    member.wakeup();
    PowerMock.expectLastCall().anyTimes();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // Poll loop for second round of calls
    member.ensureActive();
    PowerMock.expectLastCall();
    // config validation
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes();
    EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
    EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock);
    EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef());
    EasyMock.expect(connectorMock.validate(CONN1_CONFIG_UPDATED)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
    EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
    configBackingStore.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED);
    PowerMock.expectLastCall().andAnswer(new IAnswer<Object>() {

        @Override
        public Object answer() throws Throwable {
            // Simulate response to writing config + waiting until end of log to be read
            configUpdateListener.onConnectorConfigUpdate(CONN1);
            return null;
        }
    });
    // As a result of reconfig, should need to update snapshot. With only connector updates, we'll just restart
    // connector without rebalance
    EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT_UPDATED_CONN1_CONFIG);
    worker.stopConnector(CONN1);
    PowerMock.expectLastCall().andReturn(true);
    worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
    PowerMock.expectLastCall().andReturn(true);
    EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
    EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfigUpdated)).andReturn(TASK_CONFIGS);
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    // Third tick just to read the config
    member.ensureActive();
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    // Should pick up original config
    FutureCallback<Map<String, String>> connectorConfigCb = new FutureCallback<>();
    herder.connectorConfig(CONN1, connectorConfigCb);
    herder.tick();
    assertTrue(connectorConfigCb.isDone());
    assertEquals(CONN1_CONFIG, connectorConfigCb.get());
    // Apply new config.
    FutureCallback<Herder.Created<ConnectorInfo>> putConfigCb = new FutureCallback<>();
    herder.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED, true, putConfigCb);
    herder.tick();
    assertTrue(putConfigCb.isDone());
    ConnectorInfo updatedInfo = new ConnectorInfo(CONN1, CONN1_CONFIG_UPDATED, Arrays.asList(TASK0, TASK1, TASK2), ConnectorType.SOURCE);
    assertEquals(new Herder.Created<>(false, updatedInfo), putConfigCb.get());
    // Check config again to validate change
    connectorConfigCb = new FutureCallback<>();
    herder.connectorConfig(CONN1, connectorConfigCb);
    herder.tick();
    assertTrue(connectorConfigCb.isDone());
    assertEquals(CONN1_CONFIG_UPDATED, connectorConfigCb.get());
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConfigValue(org.apache.kafka.common.config.ConfigValue) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) WorkerConfig(org.apache.kafka.connect.runtime.WorkerConfig) ConnectorConfig(org.apache.kafka.connect.runtime.ConnectorConfig) Config(org.apache.kafka.common.config.Config) SinkConnectorConfig(org.apache.kafka.connect.runtime.SinkConnectorConfig) TaskConfig(org.apache.kafka.connect.runtime.TaskConfig) ConfigDef(org.apache.kafka.common.config.ConfigDef) Map(java.util.Map) HashMap(java.util.HashMap) Herder(org.apache.kafka.connect.runtime.Herder) FutureCallback(org.apache.kafka.connect.util.FutureCallback) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 32 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project apache-kafka-on-k8s by banzaicloud.

the class DistributedHerderTest method testAccessors.

@Test
public void testAccessors() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes();
    expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    member.wakeup();
    PowerMock.expectLastCall().anyTimes();
    // list connectors, get connector info, get connector config, get task configs
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    FutureCallback<Collection<String>> listConnectorsCb = new FutureCallback<>();
    herder.connectors(listConnectorsCb);
    FutureCallback<ConnectorInfo> connectorInfoCb = new FutureCallback<>();
    herder.connectorInfo(CONN1, connectorInfoCb);
    FutureCallback<Map<String, String>> connectorConfigCb = new FutureCallback<>();
    herder.connectorConfig(CONN1, connectorConfigCb);
    FutureCallback<List<TaskInfo>> taskConfigsCb = new FutureCallback<>();
    herder.taskConfigs(CONN1, taskConfigsCb);
    herder.tick();
    assertTrue(listConnectorsCb.isDone());
    assertEquals(Collections.singleton(CONN1), listConnectorsCb.get());
    assertTrue(connectorInfoCb.isDone());
    ConnectorInfo info = new ConnectorInfo(CONN1, CONN1_CONFIG, Arrays.asList(TASK0, TASK1, TASK2), ConnectorType.SOURCE);
    assertEquals(info, connectorInfoCb.get());
    assertTrue(connectorConfigCb.isDone());
    assertEquals(CONN1_CONFIG, connectorConfigCb.get());
    assertTrue(taskConfigsCb.isDone());
    assertEquals(Arrays.asList(new TaskInfo(TASK0, TASK_CONFIG), new TaskInfo(TASK1, TASK_CONFIG), new TaskInfo(TASK2, TASK_CONFIG)), taskConfigsCb.get());
    PowerMock.verifyAll();
}
Also used : TaskInfo(org.apache.kafka.connect.runtime.rest.entities.TaskInfo) ConnectorInfo(org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo) Collection(java.util.Collection) Collections.singletonList(java.util.Collections.singletonList) List(java.util.List) ArrayList(java.util.ArrayList) Map(java.util.Map) HashMap(java.util.HashMap) FutureCallback(org.apache.kafka.connect.util.FutureCallback) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 33 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerderTest method testDestroyConnector.

@Test
public void testDestroyConnector() throws Exception {
    connector = PowerMock.createMock(BogusSourceConnector.class);
    expectAdd(SourceSink.SOURCE);
    Map<String, String> config = connectorConfig(SourceSink.SOURCE);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    expectConfigValidation(connectorMock, true, config);
    EasyMock.expect(statusBackingStore.getAll(CONNECTOR_NAME)).andReturn(Collections.<TaskStatus>emptyList());
    statusBackingStore.put(new ConnectorStatus(CONNECTOR_NAME, AbstractStatus.State.DESTROYED, WORKER_ID, 0));
    expectDestroy();
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, config, false, createCallback);
    FutureCallback<Herder.Created<ConnectorInfo>> futureCb = new FutureCallback<>();
    herder.deleteConnectorConfig(CONNECTOR_NAME, futureCb);
    futureCb.get(1000L, TimeUnit.MILLISECONDS);
    // Second deletion should fail since the connector is gone
    futureCb = new FutureCallback<>();
    herder.deleteConnectorConfig(CONNECTOR_NAME, futureCb);
    try {
        futureCb.get(1000L, TimeUnit.MILLISECONDS);
        fail("Should have thrown NotFoundException");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof NotFoundException);
    }
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) WorkerConnector(org.apache.kafka.connect.runtime.WorkerConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConnectorStatus(org.apache.kafka.connect.runtime.ConnectorStatus) NotFoundException(org.apache.kafka.connect.errors.NotFoundException) ExecutionException(java.util.concurrent.ExecutionException) FutureCallback(org.apache.kafka.connect.util.FutureCallback) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 34 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerderTest method testRestartTaskFailureOnStart.

@Test
public void testRestartTaskFailureOnStart() throws Exception {
    ConnectorTaskId taskId = new ConnectorTaskId(CONNECTOR_NAME, 0);
    expectAdd(SourceSink.SOURCE);
    Map<String, String> connectorConfig = connectorConfig(SourceSink.SOURCE);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    expectConfigValidation(connectorMock, true, connectorConfig);
    worker.stopAndAwaitTask(taskId);
    EasyMock.expectLastCall();
    worker.startTask(taskId, connectorConfig, taskConfig(SourceSink.SOURCE), herder, TargetState.STARTED);
    EasyMock.expectLastCall().andReturn(false);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, connectorConfig, false, createCallback);
    FutureCallback<Void> cb = new FutureCallback<>();
    herder.restartTask(taskId, cb);
    try {
        cb.get(1000L, TimeUnit.MILLISECONDS);
        fail("Expected restart callback to raise an exception");
    } catch (ExecutionException exception) {
        assertEquals(ConnectException.class, exception.getCause().getClass());
    }
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) WorkerConnector(org.apache.kafka.connect.runtime.WorkerConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConnectorTaskId(org.apache.kafka.connect.util.ConnectorTaskId) ExecutionException(java.util.concurrent.ExecutionException) FutureCallback(org.apache.kafka.connect.util.FutureCallback) ConnectException(org.apache.kafka.connect.errors.ConnectException) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Example 35 with FutureCallback

use of org.apache.kafka.connect.util.FutureCallback in project apache-kafka-on-k8s by banzaicloud.

the class StandaloneHerderTest method testRestartTask.

@Test
public void testRestartTask() throws Exception {
    ConnectorTaskId taskId = new ConnectorTaskId(CONNECTOR_NAME, 0);
    expectAdd(SourceSink.SOURCE);
    Map<String, String> connectorConfig = connectorConfig(SourceSink.SOURCE);
    Connector connectorMock = PowerMock.createMock(SourceConnector.class);
    expectConfigValidation(connectorMock, true, connectorConfig);
    worker.stopAndAwaitTask(taskId);
    EasyMock.expectLastCall();
    worker.startTask(taskId, connectorConfig, taskConfig(SourceSink.SOURCE), herder, TargetState.STARTED);
    EasyMock.expectLastCall().andReturn(true);
    PowerMock.replayAll();
    herder.putConnectorConfig(CONNECTOR_NAME, connectorConfig, false, createCallback);
    FutureCallback<Void> cb = new FutureCallback<>();
    herder.restartTask(taskId, cb);
    cb.get(1000L, TimeUnit.MILLISECONDS);
    PowerMock.verifyAll();
}
Also used : SourceConnector(org.apache.kafka.connect.source.SourceConnector) WorkerConnector(org.apache.kafka.connect.runtime.WorkerConnector) Connector(org.apache.kafka.connect.connector.Connector) SinkConnector(org.apache.kafka.connect.sink.SinkConnector) ConnectorTaskId(org.apache.kafka.connect.util.ConnectorTaskId) FutureCallback(org.apache.kafka.connect.util.FutureCallback) PrepareForTest(org.powermock.core.classloader.annotations.PrepareForTest) Test(org.junit.Test)

Aggregations

FutureCallback (org.apache.kafka.connect.util.FutureCallback)46 Test (org.junit.Test)29 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)29 ConnectorInfo (org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo)24 Herder (org.apache.kafka.connect.runtime.Herder)23 Connector (org.apache.kafka.connect.connector.Connector)18 SinkConnector (org.apache.kafka.connect.sink.SinkConnector)18 SourceConnector (org.apache.kafka.connect.source.SourceConnector)18 ExecutionException (java.util.concurrent.ExecutionException)17 WorkerConnector (org.apache.kafka.connect.runtime.WorkerConnector)17 ConnectorTaskId (org.apache.kafka.connect.util.ConnectorTaskId)13 HashMap (java.util.HashMap)11 Path (javax.ws.rs.Path)11 RestartRequest (org.apache.kafka.connect.runtime.RestartRequest)11 ConnectorStateInfo (org.apache.kafka.connect.runtime.rest.entities.ConnectorStateInfo)11 Map (java.util.Map)9 NotFoundException (org.apache.kafka.connect.errors.NotFoundException)9 POST (javax.ws.rs.POST)7 ConnectException (org.apache.kafka.connect.errors.ConnectException)7 URI (java.net.URI)6