Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 36 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class WorkerSourceTaskTest method testTopicCreateFails.

@Test
public void testTopicCreateFails() {
    if (!enableTopicCreation)
        // should only test with topic creation enabled
        return;
    createWorkerTask();
    SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    expectPreliminaryCalls();
    EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap());
    Capture<NewTopic> newTopicCapture = EasyMock.newCapture();
    EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andThrow(new ConnectException(new TopicAuthorizationException("unauthorized")));
    PowerMock.replayAll();
    Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2));
    assertThrows(ConnectException.class, () -> Whitebox.invokeMethod(workerTask, "sendRecords"));
    assertTrue(newTopicCapture.hasCaptured());
}

Example 37 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class WorkerSourceTaskTest method testSendRecordsTopicCreateRetriesMidway.

@Test
public void testSendRecordsTopicCreateRetriesMidway() throws Exception {
    if (!enableTopicCreation)
        // should only test with topic creation enabled
        return;
    createWorkerTask();
    // Differentiate only by Kafka partition so we can reuse conversion expectations
    SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    SourceRecord record3 = new SourceRecord(PARTITION, OFFSET, OTHER_TOPIC, 3, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    // First round
    expectPreliminaryCalls(OTHER_TOPIC);
    expectTopicCreation(TOPIC);
    expectSendRecordTaskCommitRecordSucceed(false);
    expectSendRecordTaskCommitRecordSucceed(false);
    EasyMock.expect(admin.describeTopics(OTHER_TOPIC)).andReturn(Collections.emptyMap());
    // First call to create the topic times out
    Capture<NewTopic> newTopicCapture = EasyMock.newCapture();
    EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andThrow(new RetriableException(new TimeoutException("timeout")));
    // Second round
    expectTopicCreation(OTHER_TOPIC);
    expectSendRecord(OTHER_TOPIC, false, true, true, true, emptyHeaders());
    PowerMock.replayAll();
    // Try to send 3, make first pass, second fail. Should save last two
    Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2, record3));
    Whitebox.invokeMethod(workerTask, "sendRecords");
    assertEquals(Arrays.asList(record3), Whitebox.getInternalState(workerTask, "toSend"));
    // Next they all succeed
    Whitebox.invokeMethod(workerTask, "sendRecords");
    assertNull(Whitebox.getInternalState(workerTask, "toSend"));
    PowerMock.verifyAll();
}

Example 38 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class WorkerSourceTaskTest method testTopicCreateFailsWithExceptionWhenCreateReturnsTopicNotCreatedOrFound.

@Test
public void testTopicCreateFailsWithExceptionWhenCreateReturnsTopicNotCreatedOrFound() {
    if (!enableTopicCreation)
        // should only test with topic creation enabled
        return;
    createWorkerTask();
    SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    expectPreliminaryCalls();
    EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap());
    Capture<NewTopic> newTopicCapture = EasyMock.newCapture();
    EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(TopicAdmin.EMPTY_CREATION);
    PowerMock.replayAll();
    Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2));
    assertThrows(ConnectException.class, () -> Whitebox.invokeMethod(workerTask, "sendRecords"));
    assertTrue(newTopicCapture.hasCaptured());
}

Example 39 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicCreationTest method topicCreationWithOneGroupAndCombinedRegex.

@Test
public void topicCreationWithOneGroupAndCombinedRegex() {
    short fooReplicas = 3;
    int partitions = 5;
    sourceProps = defaultConnectorPropsWithTopicCreation();
    sourceProps.put(TOPIC_CREATION_GROUPS_CONFIG, String.join(",", FOO_GROUP));
    sourceProps.put(DEFAULT_TOPIC_CREATION_PREFIX + PARTITIONS_CONFIG, String.valueOf(partitions));
    // Setting here but they should be ignored for the default group
    sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + INCLUDE_REGEX_CONFIG, String.join("|", FOO_REGEX, BAR_REGEX));
    sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + REPLICATION_FACTOR_CONFIG, String.valueOf(fooReplicas));
    Map<String, String> topicProps = new HashMap<>();
    topicProps.put(CLEANUP_POLICY_CONFIG, CLEANUP_POLICY_COMPACT);
    topicProps.forEach((k, v) -> sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + k, v));
    // verify config creation
    sourceConfig = new SourceConnectorConfig(MOCK_PLUGINS, sourceProps, true);
    assertTrue(sourceConfig.usesTopicCreation());
    assertEquals(DEFAULT_REPLICATION_FACTOR, (short) sourceConfig.topicCreationReplicationFactor(DEFAULT_TOPIC_CREATION_GROUP));
    assertEquals(partitions, (int) sourceConfig.topicCreationPartitions(DEFAULT_TOPIC_CREATION_GROUP));
    assertThat(sourceConfig.topicCreationInclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.singletonList(".*")));
    assertThat(sourceConfig.topicCreationExclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyList()));
    assertThat(sourceConfig.topicCreationOtherConfigs(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyMap()));
    // verify topic creation group is instantiated correctly
    Map<String, TopicCreationGroup> groups = TopicCreationGroup.configuredGroups(sourceConfig);
    assertEquals(2, groups.size());
    assertThat(groups.keySet(), hasItems(DEFAULT_TOPIC_CREATION_GROUP, FOO_GROUP));
    // verify topic creation
    TopicCreation topicCreation = TopicCreation.newTopicCreation(workerConfig, groups);
    TopicCreationGroup defaultGroup = topicCreation.defaultTopicGroup();
    // Default group will match all topics besides empty string
    assertTrue(defaultGroup.matches(" "));
    assertTrue(defaultGroup.matches(FOO_TOPIC));
    assertTrue(defaultGroup.matches(BAR_TOPIC));
    assertEquals(DEFAULT_TOPIC_CREATION_GROUP, defaultGroup.name());
    TopicCreationGroup fooGroup = groups.get(FOO_GROUP);
    assertFalse(fooGroup.matches(" "));
    assertTrue(fooGroup.matches(FOO_TOPIC));
    assertTrue(fooGroup.matches(BAR_TOPIC));
    assertEquals(FOO_GROUP, fooGroup.name());
    assertTrue(topicCreation.isTopicCreationEnabled());
    assertTrue(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    assertTrue(topicCreation.isTopicCreationRequired(BAR_TOPIC));
    assertEquals(1, topicCreation.topicGroups().size());
    assertThat(topicCreation.topicGroups().keySet(), hasItems(FOO_GROUP));
    assertEquals(fooGroup, topicCreation.findFirstGroup(FOO_TOPIC));
    assertEquals(fooGroup, topicCreation.findFirstGroup(BAR_TOPIC));
    topicCreation.addTopic(FOO_TOPIC);
    topicCreation.addTopic(BAR_TOPIC);
    assertFalse(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    assertFalse(topicCreation.isTopicCreationRequired(BAR_TOPIC));
    // verify new topic properties
    NewTopic fooTopicSpec = topicCreation.findFirstGroup(FOO_TOPIC).newTopic(FOO_TOPIC);
    assertEquals(FOO_TOPIC, fooTopicSpec.name());
    assertEquals(fooReplicas, fooTopicSpec.replicationFactor());
    assertEquals(partitions, fooTopicSpec.numPartitions());
    assertThat(fooTopicSpec.configs(), is(topicProps));
    NewTopic barTopicSpec = topicCreation.findFirstGroup(BAR_TOPIC).newTopic(BAR_TOPIC);
    assertEquals(BAR_TOPIC, barTopicSpec.name());
    assertEquals(fooReplicas, barTopicSpec.replicationFactor());
    assertEquals(partitions, barTopicSpec.numPartitions());
    assertThat(barTopicSpec.configs(), is(topicProps));
}

Example 40 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicAdminTest method returnEmptyWithApiVersionMismatchOnCreate.

/**
 * 0.11.0.0 clients can talk with older brokers, but the CREATE_TOPIC API was added in 0.10.1.0. That means,
 * if our TopicAdmin talks to a pre 0.10.1 broker, it should receive an UnsupportedVersionException, should
 * create no topics, and return false.
 */
@Test
public void returnEmptyWithApiVersionMismatchOnCreate() {
    final NewTopic newTopic = TopicAdmin.defineTopic("myTopic").partitions(1).compacted().build();
    Cluster cluster = createCluster(1);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(new MockTime(), cluster)) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(createTopicResponseWithUnsupportedVersion(newTopic));
        TopicAdmin admin = new TopicAdmin(null, env.adminClient());
        assertTrue(admin.createOrFindTopics(newTopic).isEmpty());
    }
}