use of io.vertx.kafka.admin.NewTopic in project hono by eclipse.
the class KafkaBasedEventSenderIT method createTopics.
private static Future<Void> createTopics(final Collection<String> topicNames, final int numPartitions, final Map<String, String> topicConfig) {
topicsToDeleteAfterTests.addAll(topicNames);
final Promise<Void> resultPromise = Promise.promise();
final List<NewTopic> topics = topicNames.stream().map(t -> new NewTopic(t, numPartitions, REPLICATION_FACTOR).setConfig(topicConfig)).collect(Collectors.toList());
adminClient.createTopics(topics, resultPromise);
return resultPromise.future();
}
use of io.vertx.kafka.admin.NewTopic in project hono by eclipse.
the class HonoKafkaConsumerIT method createTopics.
private static Future<Void> createTopics(final Collection<String> topicNames, final int numPartitions, final Map<String, String> topicConfig) {
topicsToDeleteAfterTests.addAll(topicNames);
final Promise<Void> resultPromise = Promise.promise();
final List<NewTopic> topics = topicNames.stream().map(t -> new NewTopic(t, numPartitions, REPLICATION_FACTOR).setConfig(topicConfig)).collect(Collectors.toList());
adminClient.createTopics(topics, resultPromise);
return resultPromise.future();
}
use of io.vertx.kafka.admin.NewTopic in project hono by eclipse.
the class HonoKafkaConsumerIT method testConsumerReadsLatestRecordsPublishedAfterTopicSubscriptionConfirmed.
/**
* Verifies that a HonoKafkaConsumer configured with "latest" as offset reset strategy and a topic pattern
* subscription only receives records published after the consumer <em>start()</em> method has completed.
* <p>
* Also verifies that all records published after the consumer <em>ensureTopicIsAmongSubscribedTopicPatternTopics()</em>
* method has completed are received by the consumer, also if the topic was only created after the consumer
* <em>start</em> method has completed.
*
* @param partitionAssignmentStrategy The partition assignment strategy to use for the consumer.
* @param ctx The vert.x test context.
* @throws InterruptedException if test execution gets interrupted.
*/
@ParameterizedTest
@MethodSource("partitionAssignmentStrategies")
@Timeout(value = 10, timeUnit = TimeUnit.SECONDS)
public void testConsumerReadsLatestRecordsPublishedAfterTopicSubscriptionConfirmed(final String partitionAssignmentStrategy, final VertxTestContext ctx) throws InterruptedException {
final String patternPrefix = "test_" + UUID.randomUUID() + "_";
final int numTopics = 2;
final Pattern topicPattern = Pattern.compile(Pattern.quote(patternPrefix) + ".*");
final int numPartitions = 5;
final int numTestRecordsPerTopic = 20;
final Set<String> topics = IntStream.range(0, numTopics).mapToObj(i -> patternPrefix + i).collect(Collectors.toSet());
final VertxTestContext setup = new VertxTestContext();
createTopics(topics, numPartitions).compose(v -> publishRecords(numTestRecordsPerTopic, "key_", topics)).onComplete(setup.succeedingThenComplete());
assertThat(setup.awaitCompletion(IntegrationTestSupport.getTestSetupTimeout(), TimeUnit.SECONDS)).isTrue();
if (setup.failed()) {
ctx.failNow(setup.causeOfFailure());
return;
}
LOG.debug("topics created and (to be ignored) test records published");
// prepare consumer
final Map<String, String> consumerConfig = IntegrationTestSupport.getKafkaConsumerConfig().getConsumerConfig("test");
applyPartitionAssignmentStrategy(consumerConfig, partitionAssignmentStrategy);
consumerConfig.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
final AtomicReference<Promise<Void>> nextRecordReceivedPromiseRef = new AtomicReference<>();
final List<KafkaConsumerRecord<String, Buffer>> receivedRecords = new ArrayList<>();
final Handler<KafkaConsumerRecord<String, Buffer>> recordHandler = record -> {
receivedRecords.add(record);
Optional.ofNullable(nextRecordReceivedPromiseRef.get()).ifPresent(Promise::complete);
};
kafkaConsumer = new HonoKafkaConsumer(vertx, topicPattern, recordHandler, consumerConfig);
// start consumer
kafkaConsumer.start().onComplete(ctx.succeeding(v -> {
ctx.verify(() -> {
assertThat(receivedRecords.size()).isEqualTo(0);
});
final Promise<Void> nextRecordReceivedPromise = Promise.promise();
nextRecordReceivedPromiseRef.set(nextRecordReceivedPromise);
LOG.debug("consumer started, create new topic implicitly by invoking ensureTopicIsAmongSubscribedTopicPatternTopics()");
final String newTopic = patternPrefix + "new";
final String recordKey = "addedAfterStartKey";
kafkaConsumer.ensureTopicIsAmongSubscribedTopicPatternTopics(newTopic).onComplete(ctx.succeeding(v2 -> {
LOG.debug("publish record to be received by the consumer");
publish(newTopic, recordKey, Buffer.buffer("testPayload"));
}));
nextRecordReceivedPromise.future().onComplete(ar -> {
ctx.verify(() -> {
assertThat(receivedRecords.size()).isEqualTo(1);
assertThat(receivedRecords.get(0).key()).isEqualTo(recordKey);
});
ctx.completeNow();
});
}));
}
Aggregations