Examples with CreateTopicsResponseData org.apache.kafka.common.message.CreateTopicsResponseData used on opensource projects

Example 11 with CreateTopicsResponseData

use of org.apache.kafka.common.message.CreateTopicsResponseData in project kafka by apache.

the class ReplicationControlManagerTest method testGlobalTopicAndPartitionMetrics.

@Test
public void testGlobalTopicAndPartitionMetrics() throws Exception {
    ReplicationControlTestContext ctx = new ReplicationControlTestContext();
    ReplicationControlManager replicationControl = ctx.replicationControl;
    CreateTopicsRequestData request = new CreateTopicsRequestData();
    request.topics().add(new CreatableTopic().setName("foo").setNumPartitions(1).setReplicationFactor((short) -1));
    ctx.registerBrokers(0, 1, 2);
    ctx.unfenceBrokers(0, 1, 2);
    List<Uuid> topicsToDelete = new ArrayList<>();
    ControllerResult<CreateTopicsResponseData> result = replicationControl.createTopics(request);
    topicsToDelete.add(result.response().topics().find("foo").topicId());
    RecordTestUtils.replayAll(replicationControl, result.records());
    assertEquals(1, ctx.metrics.globalTopicsCount());
    request = new CreateTopicsRequestData();
    request.topics().add(new CreatableTopic().setName("bar").setNumPartitions(1).setReplicationFactor((short) -1));
    request.topics().add(new CreatableTopic().setName("baz").setNumPartitions(2).setReplicationFactor((short) -1));
    result = replicationControl.createTopics(request);
    RecordTestUtils.replayAll(replicationControl, result.records());
    assertEquals(3, ctx.metrics.globalTopicsCount());
    assertEquals(4, ctx.metrics.globalPartitionCount());
    topicsToDelete.add(result.response().topics().find("baz").topicId());
    ControllerResult<Map<Uuid, ApiError>> deleteResult = replicationControl.deleteTopics(topicsToDelete);
    RecordTestUtils.replayAll(replicationControl, deleteResult.records());
    assertEquals(1, ctx.metrics.globalTopicsCount());
    assertEquals(1, ctx.metrics.globalPartitionCount());
    Uuid topicToDelete = result.response().topics().find("bar").topicId();
    deleteResult = replicationControl.deleteTopics(Collections.singletonList(topicToDelete));
    RecordTestUtils.replayAll(replicationControl, deleteResult.records());
    assertEquals(0, ctx.metrics.globalTopicsCount());
    assertEquals(0, ctx.metrics.globalPartitionCount());
}

Example 12 with CreateTopicsResponseData

use of org.apache.kafka.common.message.CreateTopicsResponseData in project kafka by apache.

the class ReplicationControlManagerTest method testDeleteTopics.

@Test
public void testDeleteTopics() throws Exception {
    ReplicationControlTestContext ctx = new ReplicationControlTestContext();
    ReplicationControlManager replicationControl = ctx.replicationControl;
    CreateTopicsRequestData request = new CreateTopicsRequestData();
    CreateTopicsRequestData.CreateableTopicConfigCollection requestConfigs = new CreateTopicsRequestData.CreateableTopicConfigCollection();
    requestConfigs.add(new CreateTopicsRequestData.CreateableTopicConfig().setName("cleanup.policy").setValue("compact"));
    requestConfigs.add(new CreateTopicsRequestData.CreateableTopicConfig().setName("min.cleanable.dirty.ratio").setValue("0.1"));
    request.topics().add(new CreatableTopic().setName("foo").setNumPartitions(3).setReplicationFactor((short) 2).setConfigs(requestConfigs));
    ctx.registerBrokers(0, 1);
    ctx.unfenceBrokers(0, 1);
    ControllerResult<CreateTopicsResponseData> createResult = replicationControl.createTopics(request);
    CreateTopicsResponseData expectedResponse = new CreateTopicsResponseData();
    Uuid topicId = createResult.response().topics().find("foo").topicId();
    expectedResponse.topics().add(new CreatableTopicResult().setName("foo").setNumPartitions(3).setReplicationFactor((short) 2).setErrorMessage(null).setErrorCode((short) 0).setTopicId(topicId));
    assertEquals(expectedResponse, createResult.response());
    // Until the records are replayed, no changes are made
    assertNull(replicationControl.getPartition(topicId, 0));
    assertEmptyTopicConfigs(ctx, "foo");
    ctx.replay(createResult.records());
    assertNotNull(replicationControl.getPartition(topicId, 0));
    assertNotNull(replicationControl.getPartition(topicId, 1));
    assertNotNull(replicationControl.getPartition(topicId, 2));
    assertNull(replicationControl.getPartition(topicId, 3));
    assertCreatedTopicConfigs(ctx, "foo", requestConfigs);
    assertEquals(singletonMap(topicId, new ResultOrError<>("foo")), replicationControl.findTopicNames(Long.MAX_VALUE, Collections.singleton(topicId)));
    assertEquals(singletonMap("foo", new ResultOrError<>(topicId)), replicationControl.findTopicIds(Long.MAX_VALUE, Collections.singleton("foo")));
    Uuid invalidId = new Uuid(topicId.getMostSignificantBits() + 1, topicId.getLeastSignificantBits());
    assertEquals(singletonMap(invalidId, new ResultOrError<>(new ApiError(UNKNOWN_TOPIC_ID))), replicationControl.findTopicNames(Long.MAX_VALUE, Collections.singleton(invalidId)));
    assertEquals(singletonMap("bar", new ResultOrError<>(new ApiError(UNKNOWN_TOPIC_OR_PARTITION))), replicationControl.findTopicIds(Long.MAX_VALUE, Collections.singleton("bar")));
    ControllerResult<Map<Uuid, ApiError>> invalidDeleteResult = replicationControl.deleteTopics(Collections.singletonList(invalidId));
    assertEquals(0, invalidDeleteResult.records().size());
    assertEquals(singletonMap(invalidId, new ApiError(UNKNOWN_TOPIC_ID, null)), invalidDeleteResult.response());
    ControllerResult<Map<Uuid, ApiError>> deleteResult = replicationControl.deleteTopics(Collections.singletonList(topicId));
    assertTrue(deleteResult.isAtomic());
    assertEquals(singletonMap(topicId, new ApiError(NONE, null)), deleteResult.response());
    assertEquals(1, deleteResult.records().size());
    ctx.replay(deleteResult.records());
    assertNull(replicationControl.getPartition(topicId, 0));
    assertNull(replicationControl.getPartition(topicId, 1));
    assertNull(replicationControl.getPartition(topicId, 2));
    assertNull(replicationControl.getPartition(topicId, 3));
    assertEquals(singletonMap(topicId, new ResultOrError<>(new ApiError(UNKNOWN_TOPIC_ID))), replicationControl.findTopicNames(Long.MAX_VALUE, Collections.singleton(topicId)));
    assertEquals(singletonMap("foo", new ResultOrError<>(new ApiError(UNKNOWN_TOPIC_OR_PARTITION))), replicationControl.findTopicIds(Long.MAX_VALUE, Collections.singleton("foo")));
    assertEmptyTopicConfigs(ctx, "foo");
}

Example 13 with CreateTopicsResponseData

use of org.apache.kafka.common.message.CreateTopicsResponseData in project kafka by apache.

the class ReplicationControlManagerTest method testCreateTopics.

@Test
public void testCreateTopics() throws Exception {
    ReplicationControlTestContext ctx = new ReplicationControlTestContext();
    ReplicationControlManager replicationControl = ctx.replicationControl;
    CreateTopicsRequestData request = new CreateTopicsRequestData();
    request.topics().add(new CreatableTopic().setName("foo").setNumPartitions(-1).setReplicationFactor((short) -1));
    ControllerResult<CreateTopicsResponseData> result = replicationControl.createTopics(request);
    CreateTopicsResponseData expectedResponse = new CreateTopicsResponseData();
    expectedResponse.topics().add(new CreatableTopicResult().setName("foo").setErrorCode(Errors.INVALID_REPLICATION_FACTOR.code()).setErrorMessage("Unable to replicate the partition 3 time(s): All " + "brokers are currently fenced."));
    assertEquals(expectedResponse, result.response());
    ctx.registerBrokers(0, 1, 2);
    ctx.unfenceBrokers(0, 1, 2);
    ControllerResult<CreateTopicsResponseData> result2 = replicationControl.createTopics(request);
    CreateTopicsResponseData expectedResponse2 = new CreateTopicsResponseData();
    expectedResponse2.topics().add(new CreatableTopicResult().setName("foo").setNumPartitions(1).setReplicationFactor((short) 3).setErrorMessage(null).setErrorCode((short) 0).setTopicId(result2.response().topics().find("foo").topicId()));
    assertEquals(expectedResponse2, result2.response());
    ctx.replay(result2.records());
    assertEquals(new PartitionRegistration(new int[] { 1, 2, 0 }, new int[] { 1, 2, 0 }, Replicas.NONE, Replicas.NONE, 1, 0, 0), replicationControl.getPartition(((TopicRecord) result2.records().get(0).message()).topicId(), 0));
    ControllerResult<CreateTopicsResponseData> result3 = replicationControl.createTopics(request);
    CreateTopicsResponseData expectedResponse3 = new CreateTopicsResponseData();
    expectedResponse3.topics().add(new CreatableTopicResult().setName("foo").setErrorCode(Errors.TOPIC_ALREADY_EXISTS.code()).setErrorMessage("Topic 'foo' already exists."));
    assertEquals(expectedResponse3, result3.response());
    Uuid fooId = result2.response().topics().find("foo").topicId();
    RecordTestUtils.assertBatchIteratorContains(asList(asList(new ApiMessageAndVersion(new PartitionRecord().setPartitionId(0).setTopicId(fooId).setReplicas(asList(1, 2, 0)).setIsr(asList(1, 2, 0)).setRemovingReplicas(Collections.emptyList()).setAddingReplicas(Collections.emptyList()).setLeader(1).setLeaderEpoch(0).setPartitionEpoch(0), (short) 0), new ApiMessageAndVersion(new TopicRecord().setTopicId(fooId).setName("foo"), (short) 0))), ctx.replicationControl.iterator(Long.MAX_VALUE));
}

Example 14 with CreateTopicsResponseData

use of org.apache.kafka.common.message.CreateTopicsResponseData in project kafka by apache.

the class QuorumControllerTest method testSnapshotSaveAndLoad.

@Test
public void testSnapshotSaveAndLoad() throws Throwable {
    final int numBrokers = 4;
    Map<Integer, Long> brokerEpochs = new HashMap<>();
    RawSnapshotReader reader = null;
    Uuid fooId;
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.empty())) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
            QuorumController active = controlEnv.activeController();
            for (int i = 0; i < numBrokers; i++) {
                BrokerRegistrationReply reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(i).setRack(null).setClusterId(active.clusterId()).setIncarnationId(Uuid.fromString("kxAT73dKQsitIedpiPtwB" + i)).setListeners(new ListenerCollection(Arrays.asList(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092 + i)).iterator()))).get();
                brokerEpochs.put(i, reply.epoch());
            }
            for (int i = 0; i < numBrokers - 1; i++) {
                assertEquals(new BrokerHeartbeatReply(true, false, false, false), active.processBrokerHeartbeat(new BrokerHeartbeatRequestData().setWantFence(false).setBrokerEpoch(brokerEpochs.get(i)).setBrokerId(i).setCurrentMetadataOffset(100000L)).get());
            }
            CreateTopicsResponseData fooData = active.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(Arrays.asList(new CreatableReplicaAssignment().setPartitionIndex(0).setBrokerIds(Arrays.asList(0, 1, 2)), new CreatableReplicaAssignment().setPartitionIndex(1).setBrokerIds(Arrays.asList(1, 2, 0))).iterator()))).iterator()))).get();
            fooId = fooData.topics().find("foo").topicId();
            active.allocateProducerIds(new AllocateProducerIdsRequestData().setBrokerId(0).setBrokerEpoch(brokerEpochs.get(0))).get();
            long snapshotLogOffset = active.beginWritingSnapshot().get();
            reader = logEnv.waitForSnapshot(snapshotLogOffset);
            SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(reader);
            assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
            checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
        }
    }
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(3, Optional.of(reader))) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
            QuorumController active = controlEnv.activeController();
            long snapshotLogOffset = active.beginWritingSnapshot().get();
            SnapshotReader<ApiMessageAndVersion> snapshot = createSnapshotReader(logEnv.waitForSnapshot(snapshotLogOffset));
            assertEquals(snapshotLogOffset, snapshot.lastContainedLogOffset());
            checkSnapshotContent(expectedSnapshotContent(fooId, brokerEpochs), snapshot);
        }
    }
}

Example 15 with CreateTopicsResponseData

use of org.apache.kafka.common.message.CreateTopicsResponseData in project kafka by apache.

the class QuorumControllerTest method testFenceMultipleBrokers.

@Test
public void testFenceMultipleBrokers() throws Throwable {
    List<Integer> allBrokers = Arrays.asList(1, 2, 3, 4, 5);
    List<Integer> brokersToKeepUnfenced = Arrays.asList(1);
    List<Integer> brokersToFence = Arrays.asList(2, 3, 4, 5);
    short replicationFactor = 5;
    long sessionTimeoutMillis = 1000;
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1, Optional.empty());
        QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS), Optional.of(sessionTimeoutMillis))) {
        ListenerCollection listeners = new ListenerCollection();
        listeners.add(new Listener().setName("PLAINTEXT").setHost("localhost").setPort(9092));
        QuorumController active = controlEnv.activeController();
        Map<Integer, Long> brokerEpochs = new HashMap<>();
        for (Integer brokerId : allBrokers) {
            CompletableFuture<BrokerRegistrationReply> reply = active.registerBroker(new BrokerRegistrationRequestData().setBrokerId(brokerId).setClusterId(active.clusterId()).setIncarnationId(Uuid.randomUuid()).setListeners(listeners));
            brokerEpochs.put(brokerId, reply.get().epoch());
        }
        // Brokers are only registered and should still be fenced
        allBrokers.forEach(brokerId -> {
            assertFalse(active.replicationControl().isBrokerUnfenced(brokerId), "Broker " + brokerId + " should have been fenced");
        });
        // Unfence all brokers and create a topic foo
        sendBrokerheartbeat(active, allBrokers, brokerEpochs);
        CreateTopicsRequestData createTopicsRequestData = new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo").setNumPartitions(1).setReplicationFactor(replicationFactor)).iterator()));
        CreateTopicsResponseData createTopicsResponseData = active.createTopics(createTopicsRequestData).get();
        assertEquals(Errors.NONE, Errors.forCode(createTopicsResponseData.topics().find("foo").errorCode()));
        Uuid topicIdFoo = createTopicsResponseData.topics().find("foo").topicId();
        // Fence some of the brokers
        TestUtils.waitForCondition(() -> {
            sendBrokerheartbeat(active, brokersToKeepUnfenced, brokerEpochs);
            for (Integer brokerId : brokersToFence) {
                if (active.replicationControl().isBrokerUnfenced(brokerId)) {
                    return false;
                }
            }
            return true;
        }, sessionTimeoutMillis * 3, "Fencing of brokers did not process within expected time");
        // Send another heartbeat to the brokers we want to keep alive
        sendBrokerheartbeat(active, brokersToKeepUnfenced, brokerEpochs);
        // At this point only the brokers we want fenced should be fenced.
        brokersToKeepUnfenced.forEach(brokerId -> {
            assertTrue(active.replicationControl().isBrokerUnfenced(brokerId), "Broker " + brokerId + " should have been unfenced");
        });
        brokersToFence.forEach(brokerId -> {
            assertFalse(active.replicationControl().isBrokerUnfenced(brokerId), "Broker " + brokerId + " should have been fenced");
        });
        // Verify the isr and leaders for the topic partition
        int[] expectedIsr = { 1 };
        int[] isrFoo = active.replicationControl().getPartition(topicIdFoo, 0).isr;
        assertTrue(Arrays.equals(isrFoo, expectedIsr), "The ISR for topic foo was " + Arrays.toString(isrFoo) + ". It is expected to be " + Arrays.toString(expectedIsr));
        int fooLeader = active.replicationControl().getPartition(topicIdFoo, 0).leader;
        assertEquals(expectedIsr[0], fooLeader);
    }
}