use of org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection in project kafka by apache.
the class QuorumControllerTest method registerBrokers.
private Map<Integer, Long> registerBrokers(QuorumController controller, int numBrokers) throws Exception {
Map<Integer, Long> brokerEpochs = new HashMap<>();
for (int brokerId = 0; brokerId < numBrokers; brokerId++) {
BrokerRegistrationReply reply = controller.registerBroker(new BrokerRegistrationRequestData().setBrokerId(brokerId).setRack(null).setClusterId(controller.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + brokerId)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + brokerId)).iterator()))).get();
brokerEpochs.put(brokerId, reply.epoch());
// Send heartbeat to unfence
controller.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(brokerId)).setBrokerId(brokerId).setCurrentMetadataOffset(100000L)).get();
}
return brokerEpochs;
}
use of org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection in project kafka by apache.
the class QuorumControllerTest method testUnregisterBroker.
@Test
public void testUnregisterBroker() throws Throwable {
try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1, Optional.empty())) {
try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
ListenerCollection listeners = new ListenerCollection();
listeners.add(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092));
QuorumController active = controlEnv.activeController();
CompletableFuture<BrokerRegistrationReply> reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(0).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwBA")).setListeners(listeners));
assertEquals(0L, reply.get().epoch());
CreateTopicsRequestData createTopicsRequestData = new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(1).setReplicationFactor((short) 1)).iterator()));
assertEquals(Errors.INVALID_REPLICATION_FACTOR.code(), active.createTopics(createTopicsRequestData).get().topics().find("foo").errorCode());
assertEquals("Unable to replicate the partition 1 time(s): All brokers " + "are currently fenced.", active.createTopics(createTopicsRequestData).get().topics().find("foo").errorMessage());
assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(0L).setBrokerId(0).setCurrentMetadataOffset(100000L)).get());
assertEquals(Errors.NONE.code(), active.createTopics(createTopicsRequestData).get().topics().find("foo").errorCode());
CompletableFuture<TopicIdPartition> topicPartitionFuture = active.appendReadEvent("debugGetPartition", () -> {
Iterator<TopicIdPartition> iterator = active.replicationControl().brokersToIsrs().iterator(0, true);
assertTrue(iterator.hasNext());
return iterator.next();
});
assertEquals(0, topicPartitionFuture.get().partitionId());
active.unregisterBroker(0).get();
topicPartitionFuture = active.appendReadEvent("debugGetPartition", () -> {
Iterator<TopicIdPartition> iterator = active.replicationControl().brokersToIsrs().partitionsWithNoLeader();
assertTrue(iterator.hasNext());
return iterator.next();
});
assertEquals(0, topicPartitionFuture.get().partitionId());
}
}
}
use of org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection in project kafka by apache.
the class QuorumControllerTest method testSnapshotConfiguration.
@Test
public void testSnapshotConfiguration() throws Throwable {
final int numBrokers = 4;
final int maxNewRecordBytes = 4;
Map<Integer, Long> brokerEpochs = new HashMap<>();
Uuid fooId;
try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.empty())) {
try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, builder -> {
builder.setConfigDefs(CONFIGS).setSnapshotMaxNewRecordBytes(maxNewRecordBytes);
})) {
QuorumController active = controlEnv.activeController();
for (int i = 0; i < numBrokers; i++) {
BrokerRegistrationReply reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(i).setRack(null).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + i)).iterator()))).get();
brokerEpochs.put(i, reply.epoch());
}
for (int i = 0; i < numBrokers - 1; i++) {
assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)).setBrokerId(i).setCurrentMetadataOffset(100000L)).get());
}
CreateTopicsResponseData fooData = active.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(Arrays.asList(new CreatableReplicaAssignment().setPartitionIndex(0).setBrokerIds(Arrays.asList(0, 1, 2)), new CreatableReplicaAssignment().setPartitionIndex(1).setBrokerIds(Arrays.asList(1, 2, 0))).iterator()))).iterator()))).get();
fooId = fooData.topics().find("foo").topicId();
active.allocateProducerIds(new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get();
SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(logEnv.waitForLatestSnapshot());
checkSnapshotSubcontent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
}
}
}
use of org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection in project kafka by apache.
the class QuorumControllerTest method testSnapshotSaveAndLoad.
@Test
public void testSnapshotSaveAndLoad() throws Throwable {
final int numBrokers = 4;
Map<Integer, Long> brokerEpochs = new HashMap<>();
RawSnapshotReader reader = null;
Uuid fooId;
try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.empty())) {
try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
QuorumController active = controlEnv.activeController();
for (int i = 0; i < numBrokers; i++) {
BrokerRegistrationReply reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(i).setRack(null).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + i)).iterator()))).get();
brokerEpochs.put(i, reply.epoch());
}
for (int i = 0; i < numBrokers - 1; i++) {
assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)).setBrokerId(i).setCurrentMetadataOffset(100000L)).get());
}
CreateTopicsResponseData fooData = active.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(Arrays.asList(new CreatableReplicaAssignment().setPartitionIndex(0).setBrokerIds(Arrays.asList(0, 1, 2)), new CreatableReplicaAssignment().setPartitionIndex(1).setBrokerIds(Arrays.asList(1, 2, 0))).iterator()))).iterator()))).get();
fooId = fooData.topics().find("foo").topicId();
active.allocateProducerIds(new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get();
long snapshotLogOffset = active.beginWritingSnapshot().get();
reader = logEnv.waitForSnapshot(snapshotLogOffset);
SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(reader);
assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
}
}
try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.of(reader))) {
try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
QuorumController active = controlEnv.activeController();
long snapshotLogOffset = active.beginWritingSnapshot().get();
SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(logEnv.waitForSnapshot(snapshotLogOffset));
assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
}
}
}
use of org.apache.kafka.common.message.BrokerRegistrationRequestData.ListenerCollection in project kafka by apache.
the class QuorumControllerTest method testFenceMultipleBrokers.
@Test
public void testFenceMultipleBrokers() throws Throwable {
List<Integer> allBrokers = Arrays.asList(1, 2, 3, 4, 5);
List<Integer> brokersToKeepUnfenced = Arrays.asList(1);
List<Integer> brokersToFence = Arrays.asList(2, 3, 4, 5);
short replicationFactor = 5;
long sessionTimeoutMillis = 1000;
try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1, Optional.empty());
QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS), Optional.of(sessionTimeoutMillis))) {
ListenerCollection listeners = new ListenerCollection();
listeners.add(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092));
QuorumController active = controlEnv.activeController();
Map<Integer, Long> brokerEpochs = new HashMap<>();
for (Integer brokerId : allBrokers) {
CompletableFuture<BrokerRegistrationReply> reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(brokerId).setClusterId(active.clusterId()).setIncarnationId(Uuid.randomUuid()).setListeners(listeners));
brokerEpochs.put(brokerId, reply.get().epoch());
}
// Brokers are only registered and should still be fenced
allBrokers.forEach(brokerId -> {
assertFalse(active.replicationControl().isBrokerUnfenced(brokerId), "Broker " + brokerId + " should have been fenced");
});
// Unfence all brokers and create a topic foo
sendBrokerheartbeat(active, allBrokers, brokerEpochs);
CreateTopicsRequestData createTopicsRequestData = new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(1).setReplicationFactor(replicationFactor)).iterator()));
CreateTopicsResponseData createTopicsResponseData = active.createTopics(createTopicsRequestData).get();
assertEquals(Errors.NONE, Errors.forCode(createTopicsResponseData.topics().find("foo").errorCode()));
Uuid topicIdFoo = createTopicsResponseData.topics().find("foo").topicId();
// Fence some of the brokers
TestUtils.waitForCondition(() -> {
sendBrokerheartbeat(active, brokersToKeepUnfenced, brokerEpochs);
for (Integer brokerId : brokersToFence) {
if (active.replicationControl().isBrokerUnfenced(brokerId)) {
return false;
}
}
return true;
}, sessionTimeoutMillis * 3, "Fencing of brokers did not process within expected time");
// Send another heartbeat to the brokers we want to keep alive
sendBrokerheartbeat(active, brokersToKeepUnfenced, brokerEpochs);
// At this point only the brokers we want fenced should be fenced.
brokersToKeepUnfenced.forEach(brokerId -> {
assertTrue(active.replicationControl().isBrokerUnfenced(brokerId), "Broker " + brokerId + " should have been unfenced");
});
brokersToFence.forEach(brokerId -> {
assertFalse(active.replicationControl().isBrokerUnfenced(brokerId), "Broker " + brokerId + " should have been fenced");
});
// Verify the isr and leaders for the topic partition
int[] expectedIsr = { 1 };
int[] isrFoo = active.replicationControl().getPartition(topicIdFoo, 0).isr;
assertTrue(Arrays.equals(isrFoo, expectedIsr), "The ISR for topic foo was " + Arrays.toString(isrFoo) + ". It is expected to be " + Arrays.toString(expectedIsr));
int fooLeader = active.replicationControl().getPartition(topicIdFoo, 0).leader;
assertEquals(expectedIsr[0], fooLeader);
}
}
Aggregations