use of org.apache.kafka.connect.runtime.Herder in project kafka by apache.
the class DistributedHerderTest method testPutConnectorConfig.
@Test
public void testPutConnectorConfig() throws Exception {
EasyMock.expect(member.memberId()).andStubReturn("leader");
expectRebalance(1, Arrays.asList(CONN1), Collections.emptyList());
expectPostRebalanceCatchup(SNAPSHOT);
EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0);
Capture<Callback<TargetState>> onFirstStart = newCapture();
worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED), capture(onFirstStart));
PowerMock.expectLastCall().andAnswer(() -> {
onFirstStart.getValue().onCompletion(null, TargetState.STARTED);
return true;
});
EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS);
// list connectors, get connector info, get connector config, get task configs
member.wakeup();
PowerMock.expectLastCall().anyTimes();
member.poll(EasyMock.anyInt());
PowerMock.expectLastCall();
// Poll loop for second round of calls
member.ensureActive();
PowerMock.expectLastCall();
EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes();
EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT);
Capture<Callback<ConfigInfos>> validateCallback = newCapture();
herder.validateConnectorConfig(EasyMock.eq(CONN1_CONFIG_UPDATED), capture(validateCallback));
PowerMock.expectLastCall().andAnswer(() -> {
validateCallback.getValue().onCompletion(null, CONN1_CONFIG_INFOS);
return null;
});
configBackingStore.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED);
PowerMock.expectLastCall().andAnswer(() -> {
// Simulate response to writing config + waiting until end of log to be read
configUpdateListener.onConnectorConfigUpdate(CONN1);
return null;
});
// As a result of reconfig, should need to update snapshot. With only connector updates, we'll just restart
// connector without rebalance
EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT_UPDATED_CONN1_CONFIG).times(2);
worker.stopAndAwaitConnector(CONN1);
PowerMock.expectLastCall();
EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V0);
Capture<Callback<TargetState>> onSecondStart = newCapture();
worker.startConnector(EasyMock.eq(CONN1), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED), capture(onSecondStart));
PowerMock.expectLastCall().andAnswer(() -> {
onSecondStart.getValue().onCompletion(null, TargetState.STARTED);
return true;
});
EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfigUpdated)).andReturn(TASK_CONFIGS);
member.poll(EasyMock.anyInt());
PowerMock.expectLastCall();
// Third tick just to read the config
member.ensureActive();
PowerMock.expectLastCall();
member.poll(EasyMock.anyInt());
PowerMock.expectLastCall();
PowerMock.replayAll();
// Should pick up original config
FutureCallback<Map<String, String>> connectorConfigCb = new FutureCallback<>();
herder.connectorConfig(CONN1, connectorConfigCb);
herder.tick();
assertTrue(connectorConfigCb.isDone());
assertEquals(CONN1_CONFIG, connectorConfigCb.get());
// Apply new config.
FutureCallback<Herder.Created<ConnectorInfo>> putConfigCb = new FutureCallback<>();
herder.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED, true, putConfigCb);
herder.tick();
assertTrue(putConfigCb.isDone());
ConnectorInfo updatedInfo = new ConnectorInfo(CONN1, CONN1_CONFIG_UPDATED, Arrays.asList(TASK0, TASK1, TASK2), ConnectorType.SOURCE);
assertEquals(new Herder.Created<>(false, updatedInfo), putConfigCb.get());
// Check config again to validate change
connectorConfigCb = new FutureCallback<>();
herder.connectorConfig(CONN1, connectorConfigCb);
herder.tick();
assertTrue(connectorConfigCb.isDone());
assertEquals(CONN1_CONFIG_UPDATED, connectorConfigCb.get());
PowerMock.verifyAll();
}
use of org.apache.kafka.connect.runtime.Herder in project apache-kafka-on-k8s by banzaicloud.
the class DistributedHerderTest method testPutConnectorConfig.
@Test
public void testPutConnectorConfig() throws Exception {
EasyMock.expect(member.memberId()).andStubReturn("leader");
expectRebalance(1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList());
expectPostRebalanceCatchup(SNAPSHOT);
worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
PowerMock.expectLastCall().andReturn(true);
EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfig)).andReturn(TASK_CONFIGS);
// list connectors, get connector info, get connector config, get task configs
member.wakeup();
PowerMock.expectLastCall().anyTimes();
member.poll(EasyMock.anyInt());
PowerMock.expectLastCall();
// Poll loop for second round of calls
member.ensureActive();
PowerMock.expectLastCall();
// config validation
Connector connectorMock = PowerMock.createMock(SourceConnector.class);
EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes();
EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader);
EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock);
EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef());
EasyMock.expect(connectorMock.validate(CONN1_CONFIG_UPDATED)).andReturn(new Config(Collections.<ConfigValue>emptyList()));
EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader);
configBackingStore.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED);
PowerMock.expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
// Simulate response to writing config + waiting until end of log to be read
configUpdateListener.onConnectorConfigUpdate(CONN1);
return null;
}
});
// As a result of reconfig, should need to update snapshot. With only connector updates, we'll just restart
// connector without rebalance
EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT_UPDATED_CONN1_CONFIG);
worker.stopConnector(CONN1);
PowerMock.expectLastCall().andReturn(true);
worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
PowerMock.expectLastCall().andReturn(true);
EasyMock.expect(worker.isRunning(CONN1)).andReturn(true);
EasyMock.expect(worker.connectorTaskConfigs(CONN1, conn1SinkConfigUpdated)).andReturn(TASK_CONFIGS);
member.poll(EasyMock.anyInt());
PowerMock.expectLastCall();
// Third tick just to read the config
member.ensureActive();
PowerMock.expectLastCall();
member.poll(EasyMock.anyInt());
PowerMock.expectLastCall();
PowerMock.replayAll();
// Should pick up original config
FutureCallback<Map<String, String>> connectorConfigCb = new FutureCallback<>();
herder.connectorConfig(CONN1, connectorConfigCb);
herder.tick();
assertTrue(connectorConfigCb.isDone());
assertEquals(CONN1_CONFIG, connectorConfigCb.get());
// Apply new config.
FutureCallback<Herder.Created<ConnectorInfo>> putConfigCb = new FutureCallback<>();
herder.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED, true, putConfigCb);
herder.tick();
assertTrue(putConfigCb.isDone());
ConnectorInfo updatedInfo = new ConnectorInfo(CONN1, CONN1_CONFIG_UPDATED, Arrays.asList(TASK0, TASK1, TASK2), ConnectorType.SOURCE);
assertEquals(new Herder.Created<>(false, updatedInfo), putConfigCb.get());
// Check config again to validate change
connectorConfigCb = new FutureCallback<>();
herder.connectorConfig(CONN1, connectorConfigCb);
herder.tick();
assertTrue(connectorConfigCb.isDone());
assertEquals(CONN1_CONFIG_UPDATED, connectorConfigCb.get());
PowerMock.verifyAll();
}
use of org.apache.kafka.connect.runtime.Herder in project apache-kafka-on-k8s by banzaicloud.
the class StandaloneHerderTest method testPutConnectorConfig.
@Test
public void testPutConnectorConfig() throws Exception {
Map<String, String> connConfig = connectorConfig(SourceSink.SOURCE);
Map<String, String> newConnConfig = new HashMap<>(connConfig);
newConnConfig.put("foo", "bar");
Callback<Map<String, String>> connectorConfigCb = PowerMock.createMock(Callback.class);
Callback<Herder.Created<ConnectorInfo>> putConnectorConfigCb = PowerMock.createMock(Callback.class);
// Create
connector = PowerMock.createMock(BogusSourceConnector.class);
expectAdd(SourceSink.SOURCE);
Connector connectorMock = PowerMock.createMock(SourceConnector.class);
expectConfigValidation(connectorMock, true, connConfig);
// Should get first config
connectorConfigCb.onCompletion(null, connConfig);
EasyMock.expectLastCall();
// Update config, which requires stopping and restarting
worker.stopConnector(CONNECTOR_NAME);
EasyMock.expectLastCall().andReturn(true);
Capture<Map<String, String>> capturedConfig = EasyMock.newCapture();
worker.startConnector(EasyMock.eq(CONNECTOR_NAME), EasyMock.capture(capturedConfig), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED));
EasyMock.expectLastCall().andReturn(true);
EasyMock.expect(worker.isRunning(CONNECTOR_NAME)).andReturn(true);
// Generate same task config, which should result in no additional action to restart tasks
EasyMock.expect(worker.connectorTaskConfigs(CONNECTOR_NAME, new SourceConnectorConfig(plugins, newConnConfig))).andReturn(singletonList(taskConfig(SourceSink.SOURCE)));
worker.isSinkConnector(CONNECTOR_NAME);
EasyMock.expectLastCall().andReturn(false);
ConnectorInfo newConnInfo = new ConnectorInfo(CONNECTOR_NAME, newConnConfig, Arrays.asList(new ConnectorTaskId(CONNECTOR_NAME, 0)), ConnectorType.SOURCE);
putConnectorConfigCb.onCompletion(null, new Herder.Created<>(false, newConnInfo));
EasyMock.expectLastCall();
// Should get new config
expectConfigValidation(connectorMock, false, newConnConfig);
connectorConfigCb.onCompletion(null, newConnConfig);
EasyMock.expectLastCall();
EasyMock.expect(worker.getPlugins()).andReturn(plugins).anyTimes();
PowerMock.replayAll();
herder.putConnectorConfig(CONNECTOR_NAME, connConfig, false, createCallback);
herder.connectorConfig(CONNECTOR_NAME, connectorConfigCb);
herder.putConnectorConfig(CONNECTOR_NAME, newConnConfig, true, putConnectorConfigCb);
assertEquals("bar", capturedConfig.getValue().get("foo"));
herder.connectorConfig(CONNECTOR_NAME, connectorConfigCb);
PowerMock.verifyAll();
}
use of org.apache.kafka.connect.runtime.Herder in project kafka by apache.
the class MirrorMaker method addHerder.
private void addHerder(SourceAndTarget sourceAndTarget) {
log.info("creating herder for " + sourceAndTarget.toString());
Map<String, String> workerProps = config.workerConfig(sourceAndTarget);
String advertisedUrl = advertisedBaseUrl + "/" + sourceAndTarget.source();
String workerId = sourceAndTarget.toString();
Plugins plugins = new Plugins(workerProps);
plugins.compareAndSwapWithDelegatingLoader();
DistributedConfig distributedConfig = new DistributedConfig(workerProps);
String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(distributedConfig);
// Create the admin client to be shared by all backing stores for this herder
Map<String, Object> adminProps = new HashMap<>(distributedConfig.originals());
ConnectUtils.addMetricsContextProperties(adminProps, distributedConfig, kafkaClusterId);
SharedTopicAdmin sharedAdmin = new SharedTopicAdmin(adminProps);
KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(sharedAdmin);
offsetBackingStore.configure(distributedConfig);
Worker worker = new Worker(workerId, time, plugins, distributedConfig, offsetBackingStore, CLIENT_CONFIG_OVERRIDE_POLICY);
WorkerConfigTransformer configTransformer = worker.configTransformer();
Converter internalValueConverter = worker.getInternalValueConverter();
StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter, sharedAdmin);
statusBackingStore.configure(distributedConfig);
ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(internalValueConverter, distributedConfig, configTransformer, sharedAdmin);
// Pass the shared admin to the distributed herder as an additional AutoCloseable object that should be closed when the
// herder is stopped. MirrorMaker has multiple herders, and having the herder own the close responsibility is much easier than
// tracking the various shared admin objects in this class.
Herder herder = new DistributedHerder(distributedConfig, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, advertisedUrl, CLIENT_CONFIG_OVERRIDE_POLICY, sharedAdmin);
herders.put(sourceAndTarget, herder);
}
use of org.apache.kafka.connect.runtime.Herder in project kafka by apache.
the class MirrorMaker method start.
public void start() {
log.info("Kafka MirrorMaker starting with {} herders.", herders.size());
if (startLatch != null) {
throw new IllegalStateException("MirrorMaker instance already started");
}
startLatch = new CountDownLatch(herders.size());
stopLatch = new CountDownLatch(herders.size());
Exit.addShutdownHook("mirror-maker-shutdown-hook", shutdownHook);
for (Herder herder : herders.values()) {
try {
herder.start();
} finally {
startLatch.countDown();
}
}
log.info("Configuring connectors...");
herderPairs.forEach(this::configureConnectors);
log.info("Kafka MirrorMaker started");
}
Aggregations