Examples with BrokerRegistrationRequestData org.apache.kafka.common.message.BrokerRegistrationRequestData used on opensource projects

Example 6 with BrokerRegistrationRequestData

use of org.apache.kafka.common.message.BrokerRegistrationRequestData in project kafka by apache.

the class QuorumControllerTest method testSnapshotSaveAndLoad.

@Test
public void testSnapshotSaveAndLoad() throws Throwable {
    final int numBrokers = 4;
    Map<Integer, Long> brokerEpochs = new HashMap<>();
    RawSnapshotReader reader = null;
    Uuid fooId;
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.empty())) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
            QuorumController active = controlEnv.activeController();
            for (int i = 0; i < numBrokers; i++) {
                BrokerRegistrationReply reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(i).setRack(null).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + i)).iterator()))).get();
                brokerEpochs.put(i, reply.epoch());
            }
            for (int i = 0; i < numBrokers - 1; i++) {
                assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)).setBrokerId(i).setCurrentMetadataOffset(100000L)).get());
            }
            CreateTopicsResponseData fooData = active.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(Arrays.asList(new CreatableReplicaAssignment().setPartitionIndex(0).setBrokerIds(Arrays.asList(0, 1, 2)), new CreatableReplicaAssignment().setPartitionIndex(1).setBrokerIds(Arrays.asList(1, 2, 0))).iterator()))).iterator()))).get();
            fooId = fooData.topics().find("foo").topicId();
            active.allocateProducerIds(new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get();
            long snapshotLogOffset = active.beginWritingSnapshot().get();
            reader = logEnv.waitForSnapshot(snapshotLogOffset);
            SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(reader);
            assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
            checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
        }
    }
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.of(reader))) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
            QuorumController active = controlEnv.activeController();
            long snapshotLogOffset = active.beginWritingSnapshot().get();
            SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(logEnv.waitForSnapshot(snapshotLogOffset));
            assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
            checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
        }
    }
}

Example 7 with BrokerRegistrationRequestData

use of org.apache.kafka.common.message.BrokerRegistrationRequestData in project kafka by apache.

the class QuorumControllerTest method testFenceMultipleBrokers.

@Test
public void testFenceMultipleBrokers() throws Throwable {
    List<Integer> allBrokers = Arrays.asList(1, 2, 3, 4, 5);
    List<Integer> brokersToKeepUnfenced = Arrays.asList(1);
    List<Integer> brokersToFence = Arrays.asList(2, 3, 4, 5);
    short replicationFactor = 5;
    long sessionTimeoutMillis = 1000;
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1, Optional.empty());
        QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS), Optional.of(sessionTimeoutMillis))) {
        ListenerCollection listeners = new ListenerCollection();
        listeners.add(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092));
        QuorumController active = controlEnv.activeController();
        Map<Integer, Long> brokerEpochs = new HashMap<>();
        for (Integer brokerId : allBrokers) {
            CompletableFuture<BrokerRegistrationReply> reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(brokerId).setClusterId(active.clusterId()).setIncarnationId(Uuid.randomUuid()).setListeners(listeners));
            brokerEpochs.put(brokerId, reply.get().epoch());
        }
        // Brokers are only registered and should still be fenced
        allBrokers.forEach(brokerId -> {
            assertFalse(active.replicationControl().isBrokerUnfenced(brokerId), "Broker " + brokerId + " should have been fenced");
        });
        // Unfence all brokers and create a topic foo
        sendBrokerheartbeat(active, allBrokers, brokerEpochs);
        CreateTopicsRequestData createTopicsRequestData = new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(1).setReplicationFactor(replicationFactor)).iterator()));
        CreateTopicsResponseData createTopicsResponseData = active.createTopics(createTopicsRequestData).get();
        assertEquals(Errors.NONE, Errors.forCode(createTopicsResponseData.topics().find("foo").errorCode()));
        Uuid topicIdFoo = createTopicsResponseData.topics().find("foo").topicId();
        // Fence some of the brokers
        TestUtils.waitForCondition(() -> {
            sendBrokerheartbeat(active, brokersToKeepUnfenced, brokerEpochs);
            for (Integer brokerId : brokersToFence) {
                if (active.replicationControl().isBrokerUnfenced(brokerId)) {
                    return false;
                }
            }
            return true;
        }, sessionTimeoutMillis * 3, "Fencing of brokers did not process within expected time");
        // Send another heartbeat to the brokers we want to keep alive
        sendBrokerheartbeat(active, brokersToKeepUnfenced, brokerEpochs);
        // At this point only the brokers we want fenced should be fenced.
        brokersToKeepUnfenced.forEach(brokerId -> {
            assertTrue(active.replicationControl().isBrokerUnfenced(brokerId), "Broker " + brokerId + " should have been unfenced");
        });
        brokersToFence.forEach(brokerId -> {
            assertFalse(active.replicationControl().isBrokerUnfenced(brokerId), "Broker " + brokerId + " should have been fenced");
        });
        // Verify the isr and leaders for the topic partition
        int[] expectedIsr = { 1 };
        int[] isrFoo = active.replicationControl().getPartition(topicIdFoo, 0).isr;
        assertTrue(Arrays.equals(isrFoo, expectedIsr), "The ISR for topic foo was " + Arrays.toString(isrFoo) + ". It is expected to be " + Arrays.toString(expectedIsr));
        int fooLeader = active.replicationControl().getPartition(topicIdFoo, 0).leader;
        assertEquals(expectedIsr[0], fooLeader);
    }
}

Example 8 with BrokerRegistrationRequestData

use of org.apache.kafka.common.message.BrokerRegistrationRequestData in project kafka by apache.

the class QuorumControllerTest method testSnapshotOnlyAfterConfiguredMinBytes.

@Test
public void testSnapshotOnlyAfterConfiguredMinBytes() throws Throwable {
    final int numBrokers = 4;
    final int maxNewRecordBytes = 1000;
    Map<Integer, Long> brokerEpochs = new HashMap<>();
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.empty())) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, builder -> builder.setConfigDefs(CONFIGS).setSnapshotMaxNewRecordBytes(maxNewRecordBytes))) {
            QuorumController active = controlEnv.activeController();
            for (int i = 0; i < numBrokers; i++) {
                BrokerRegistrationReply reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(i).setRack(null).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + i)).iterator()))).get();
                brokerEpochs.put(i, reply.epoch());
                assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)).setBrokerId(i).setCurrentMetadataOffset(100000L)).get());
            }
            assertTrue(logEnv.appendedBytes() < maxNewRecordBytes, String.format("%s appended bytes is not less than %s max new record bytes", logEnv.appendedBytes(), maxNewRecordBytes));
            // Keep creating topic until we reached the max bytes limit
            int counter = 0;
            while (logEnv.appendedBytes() < maxNewRecordBytes) {
                counter += 1;
                String topicName = String.format("foo-%s", counter);
                active.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName(topicName).setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(Arrays.asList(new CreatableReplicaAssignment().setPartitionIndex(0).setBrokerIds(Arrays.asList(0, 1, 2)), new CreatableReplicaAssignment().setPartitionIndex(1).setBrokerIds(Arrays.asList(1, 2, 0))).iterator()))).iterator()))).get();
            }
            logEnv.waitForLatestSnapshot();
        }
    }
}

Example 9 with BrokerRegistrationRequestData

use of org.apache.kafka.common.message.BrokerRegistrationRequestData in project kafka by apache.

the class ClusterControlManagerTest method testRegistrationWithIncorrectClusterId.

@Test
public void testRegistrationWithIncorrectClusterId() throws Exception {
    SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext());
    ClusterControlManager clusterControl = new ClusterControlManager(new LogContext(), "fPZv1VBsRFmnlRvmGcOW9w", new MockTime(0, 0, 0), snapshotRegistry, 1000, new StripedReplicaPlacer(new Random()), new MockControllerMetrics());
    clusterControl.activate();
    assertThrows(InconsistentClusterIdException.class, () -> clusterControl.registerBroker(new BrokerRegistrationRequestData().setClusterId("WIjw3grwRZmR2uOpdpVXbg").setBrokerId(0).setRack(null).setIncarnationId(Uuid.fromString("0H4fUu1xQEKXFYwB1aBjhg")), 123L, new FeatureMapAndEpoch(new FeatureMap(Collections.emptyMap()), 456L)));
}

Example 10 with BrokerRegistrationRequestData

use of org.apache.kafka.common.message.BrokerRegistrationRequestData in project kafka by apache.

the class ClusterControlManager method registerBroker.

/**
 * Process an incoming broker registration request.
 */
public ControllerResult<BrokerRegistrationReply> registerBroker(BrokerRegistrationRequestData request, long brokerEpoch, FeatureMapAndEpoch finalizedFeatures) {
    if (heartbeatManager == null) {
        throw new RuntimeException("ClusterControlManager is not active.");
    }
    if (!clusterId.equals(request.clusterId())) {
        throw new InconsistentClusterIdException("Expected cluster ID " + clusterId + ", but got cluster ID " + request.clusterId());
    }
    int brokerId = request.brokerId();
    BrokerRegistration existing = brokerRegistrations.get(brokerId);
    if (existing != null) {
        if (heartbeatManager.hasValidSession(brokerId)) {
            if (!existing.incarnationId().equals(request.incarnationId())) {
                throw new DuplicateBrokerRegistrationException("Another broker is " + "registered with that broker id.");
            }
        } else {
            if (!existing.incarnationId().equals(request.incarnationId())) {
                // Remove any existing session for the old broker incarnation.
                heartbeatManager.remove(brokerId);
                existing = null;
            }
        }
    }
    RegisterBrokerRecord record = new RegisterBrokerRecord().setBrokerId(brokerId).setIncarnationId(request.incarnationId()).setBrokerEpoch(brokerEpoch).setRack(request.rack());
    for (BrokerRegistrationRequestData.Listener listener : request.listeners()) {
        record.endPoints().add(new BrokerEndpoint().setHost(listener.host()).setName(listener.name()).setPort(listener.port()).setSecurityProtocol(listener.securityProtocol()));
    }
    for (BrokerRegistrationRequestData.Feature feature : request.features()) {
        Optional<VersionRange> finalized = finalizedFeatures.map().get(feature.name());
        if (finalized.isPresent()) {
            if (!finalized.get().contains(new VersionRange(feature.minSupportedVersion(), feature.maxSupportedVersion()))) {
                throw new UnsupportedVersionException("Unable to register because " + "the broker has an unsupported version of " + feature.name());
            }
        }
        record.features().add(new BrokerFeature().setName(feature.name()).setMinSupportedVersion(feature.minSupportedVersion()).setMaxSupportedVersion(feature.maxSupportedVersion()));
    }
    if (existing == null) {
        heartbeatManager.touch(brokerId, true, -1);
    } else {
        heartbeatManager.touch(brokerId, existing.fenced(), -1);
    }
    List<ApiMessageAndVersion> records = new ArrayList<>();
    records.add(new ApiMessageAndVersion(record, REGISTER_BROKER_RECORD.highestSupportedVersion()));
    return ControllerResult.atomicOf(records, new BrokerRegistrationReply(brokerEpoch));
}