Examples with NewTopic org.apache.kafka.clients.admin.NewTopic used on opensource projects

Example 46 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class TopicCreationTest method topicCreationWithTwoGroupsAndTwoTransformations.

@Test
public void topicCreationWithTwoGroupsAndTwoTransformations() {
    short fooReplicas = 3;
    int partitions = 5;
    int barPartitions = 1;
    sourceProps = defaultConnectorPropsWithTopicCreation();
    sourceProps.put(TOPIC_CREATION_GROUPS_CONFIG, String.join(",", FOO_GROUP, BAR_GROUP));
    sourceProps.put(DEFAULT_TOPIC_CREATION_PREFIX + PARTITIONS_CONFIG, String.valueOf(partitions));
    // Setting here but they should be ignored for the default group
    sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + INCLUDE_REGEX_CONFIG, FOO_TOPIC);
    sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + REPLICATION_FACTOR_CONFIG, String.valueOf(fooReplicas));
    sourceProps.put(TOPIC_CREATION_PREFIX + BAR_GROUP + "." + INCLUDE_REGEX_CONFIG, BAR_REGEX);
    sourceProps.put(TOPIC_CREATION_PREFIX + BAR_GROUP + "." + PARTITIONS_CONFIG, String.valueOf(barPartitions));
    String castName = "cast";
    String castType = "int8";
    sourceProps.put("transforms." + castName + ".type", Cast.Value.class.getName());
    sourceProps.put("transforms." + castName + ".spec", castType);
    String regexRouterName = "regex";
    sourceProps.put("transforms." + regexRouterName + ".type", RegexRouter.class.getName());
    sourceProps.put("transforms." + regexRouterName + ".regex", "(.*)");
    sourceProps.put("transforms." + regexRouterName + ".replacement", "prefix-$1");
    sourceProps.put("transforms", String.join(",", castName, regexRouterName));
    Map<String, String> fooTopicProps = new HashMap<>();
    fooTopicProps.put(RETENTION_MS_CONFIG, String.valueOf(TimeUnit.DAYS.toMillis(30)));
    fooTopicProps.forEach((k, v) -> sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + k, v));
    Map<String, String> barTopicProps = new HashMap<>();
    barTopicProps.put(CLEANUP_POLICY_CONFIG, CLEANUP_POLICY_COMPACT);
    barTopicProps.forEach((k, v) -> sourceProps.put(TOPIC_CREATION_PREFIX + BAR_GROUP + "." + k, v));
    // verify config creation
    sourceConfig = new SourceConnectorConfig(MOCK_PLUGINS, sourceProps, true);
    assertTrue(sourceConfig.usesTopicCreation());
    assertEquals(DEFAULT_REPLICATION_FACTOR, (short) sourceConfig.topicCreationReplicationFactor(DEFAULT_TOPIC_CREATION_GROUP));
    assertEquals(partitions, (int) sourceConfig.topicCreationPartitions(DEFAULT_TOPIC_CREATION_GROUP));
    assertThat(sourceConfig.topicCreationInclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.singletonList(".*")));
    assertThat(sourceConfig.topicCreationExclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyList()));
    assertThat(sourceConfig.topicCreationOtherConfigs(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyMap()));
    // verify topic creation group is instantiated correctly
    Map<String, TopicCreationGroup> groups = TopicCreationGroup.configuredGroups(sourceConfig);
    assertEquals(3, groups.size());
    assertThat(groups.keySet(), hasItems(DEFAULT_TOPIC_CREATION_GROUP, FOO_GROUP, BAR_GROUP));
    // verify topic creation
    TopicCreation topicCreation = TopicCreation.newTopicCreation(workerConfig, groups);
    TopicCreationGroup defaultGroup = topicCreation.defaultTopicGroup();
    // Default group will match all topics besides empty string
    assertTrue(defaultGroup.matches(" "));
    assertTrue(defaultGroup.matches(FOO_TOPIC));
    assertTrue(defaultGroup.matches(BAR_TOPIC));
    assertEquals(DEFAULT_TOPIC_CREATION_GROUP, defaultGroup.name());
    TopicCreationGroup fooGroup = groups.get(FOO_GROUP);
    assertFalse(fooGroup.matches(" "));
    assertTrue(fooGroup.matches(FOO_TOPIC));
    assertFalse(fooGroup.matches(BAR_TOPIC));
    assertEquals(FOO_GROUP, fooGroup.name());
    TopicCreationGroup barGroup = groups.get(BAR_GROUP);
    assertTrue(barGroup.matches(BAR_TOPIC));
    assertFalse(barGroup.matches(FOO_TOPIC));
    assertEquals(BAR_GROUP, barGroup.name());
    assertTrue(topicCreation.isTopicCreationEnabled());
    assertTrue(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    assertTrue(topicCreation.isTopicCreationRequired(BAR_TOPIC));
    assertEquals(2, topicCreation.topicGroups().size());
    assertThat(topicCreation.topicGroups().keySet(), hasItems(FOO_GROUP, BAR_GROUP));
    assertEquals(fooGroup, topicCreation.findFirstGroup(FOO_TOPIC));
    assertEquals(barGroup, topicCreation.findFirstGroup(BAR_TOPIC));
    topicCreation.addTopic(FOO_TOPIC);
    topicCreation.addTopic(BAR_TOPIC);
    assertFalse(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    assertFalse(topicCreation.isTopicCreationRequired(BAR_TOPIC));
    // verify new topic properties
    String otherTopic = "any-other-topic";
    NewTopic defaultTopicSpec = topicCreation.findFirstGroup(otherTopic).newTopic(otherTopic);
    assertEquals(otherTopic, defaultTopicSpec.name());
    assertEquals(DEFAULT_REPLICATION_FACTOR, defaultTopicSpec.replicationFactor());
    assertEquals(partitions, defaultTopicSpec.numPartitions());
    assertThat(defaultTopicSpec.configs(), is(Collections.emptyMap()));
    NewTopic fooTopicSpec = topicCreation.findFirstGroup(FOO_TOPIC).newTopic(FOO_TOPIC);
    assertEquals(FOO_TOPIC, fooTopicSpec.name());
    assertEquals(fooReplicas, fooTopicSpec.replicationFactor());
    assertEquals(partitions, fooTopicSpec.numPartitions());
    assertThat(fooTopicSpec.configs(), is(fooTopicProps));
    NewTopic barTopicSpec = topicCreation.findFirstGroup(BAR_TOPIC).newTopic(BAR_TOPIC);
    assertEquals(BAR_TOPIC, barTopicSpec.name());
    assertEquals(DEFAULT_REPLICATION_FACTOR, barTopicSpec.replicationFactor());
    assertEquals(barPartitions, barTopicSpec.numPartitions());
    assertThat(barTopicSpec.configs(), is(barTopicProps));
    List<Transformation<SourceRecord>> transformations = sourceConfig.transformations();
    assertEquals(2, transformations.size());
    Cast<SourceRecord> castXForm = (Cast<SourceRecord>) transformations.get(0);
    SourceRecord transformed = castXForm.apply(new SourceRecord(null, null, "topic", 0, null, null, Schema.INT8_SCHEMA, 42));
    assertEquals(Schema.Type.INT8, transformed.valueSchema().type());
    assertEquals((byte) 42, transformed.value());
    RegexRouter<SourceRecord> regexRouterXForm = (RegexRouter<SourceRecord>) transformations.get(1);
    transformed = regexRouterXForm.apply(new SourceRecord(null, null, "topic", 0, null, null, Schema.INT8_SCHEMA, 42));
    assertEquals("prefix-topic", transformed.topic());
}

Example 47 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class KafkaEmbedded method createTopic.

/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(final String topic, final int partitions, final int replication, final Map<String, String> topicConfig) {
    log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }", topic, partitions, replication, topicConfig);
    final NewTopic newTopic = new NewTopic(topic, partitions, (short) replication);
    newTopic.configs(topicConfig);
    try (final Admin adminClient = createAdminClient()) {
        adminClient.createTopics(Collections.singletonList(newTopic)).all().get();
    } catch (final InterruptedException | ExecutionException e) {
        throw new RuntimeException(e);
    }
}

Example 48 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project kafka by apache.

the class WorkerSourceTaskTest method testTopicCreateSucceedsWhenCreateReturnsNewTopicFound.

@Test
public void testTopicCreateSucceedsWhenCreateReturnsNewTopicFound() throws Exception {
    if (!enableTopicCreation)
        // should only test with topic creation enabled
        return;
    createWorkerTask();
    SourceRecord record1 = new SourceRecord(PARTITION, OFFSET, TOPIC, 1, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    SourceRecord record2 = new SourceRecord(PARTITION, OFFSET, TOPIC, 2, KEY_SCHEMA, KEY, RECORD_SCHEMA, RECORD);
    expectPreliminaryCalls();
    EasyMock.expect(admin.describeTopics(TOPIC)).andReturn(Collections.emptyMap());
    Capture<NewTopic> newTopicCapture = EasyMock.newCapture();
    EasyMock.expect(admin.createOrFindTopics(EasyMock.capture(newTopicCapture))).andReturn(createdTopic(TOPIC));
    expectSendRecordTaskCommitRecordSucceed(false);
    expectSendRecordTaskCommitRecordSucceed(false);
    PowerMock.replayAll();
    Whitebox.setInternalState(workerTask, "toSend", Arrays.asList(record1, record2));
    Whitebox.invokeMethod(workerTask, "sendRecords");
}

Example 49 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project druid by druid-io.

the class KafkaSupervisorTest method addSomeEvents.

private void addSomeEvents(int numEventsPerPartition) throws Exception {
    // create topic manually
    try (Admin admin = kafkaServer.newAdminClient()) {
        admin.createTopics(Collections.singletonList(new NewTopic(topic, NUM_PARTITIONS, (short) 1))).all().get();
    }
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        kafkaProducer.initTransactions();
        kafkaProducer.beginTransaction();
        for (int i = 0; i < NUM_PARTITIONS; i++) {
            for (int j = 0; j < numEventsPerPartition; j++) {
                kafkaProducer.send(new ProducerRecord<>(topic, i, null, StringUtils.toUtf8(StringUtils.format("event-%d", j)))).get();
            }
        }
        kafkaProducer.commitTransaction();
    }
}

Example 50 with NewTopic

use of org.apache.kafka.clients.admin.NewTopic in project hive by apache.

the class SingleNodeKafkaCluster method createTopic.

private void createTopic(String topic) {
    Properties properties = new Properties();
    properties.setProperty("bootstrap.servers", "localhost:9092");
    properties.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    properties.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    int numPartitions = 1;
    short replicationFactor = 1;
    AdminClient adminClient = AdminClient.create(properties);
    NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor);
    adminClient.createTopics(Collections.singletonList(newTopic));
    adminClient.close();
}