use of io.vertx.kafka.client.consumer.KafkaConsumerRecord in project hono by eclipse.
the class KafkaBasedCommandConsumerFactoryImplIT method testCommandsGetForwardedIfOneConsumerInstanceGetsClosed.
/**
* Verifies that records, published on the tenant-specific Kafka command topic, get received
* and forwarded by consumers created by factory instances even if one factory and its contained
* consumer gets closed in the middle of processing some of the commands.
*
* @param ctx The vert.x test context.
* @throws InterruptedException if test execution gets interrupted.
*/
@Test
@Timeout(value = 10, timeUnit = TimeUnit.SECONDS)
public void testCommandsGetForwardedIfOneConsumerInstanceGetsClosed(final VertxTestContext ctx) throws InterruptedException {
final String tenantId = "tenant_" + UUID.randomUUID();
final VertxTestContext setup = new VertxTestContext();
// Scenario to test:
// - first command gets sent, forwarded and received without any imposed delay
// - second command gets sent, received by the factory consumer instance; processing gets blocked
// while trying to get the target adapter instance
// - for the rest of the commands, retrieval of the target adapter instance is successful, but they won't
// get forwarded until processing of the second command is finished
// - now the factory consumer gets closed and a new factory/consumer gets started; at that point
// also the processing of the second command gets finished
//
// Expected outcome:
// - processing of the second command and all following commands by the first consumer gets aborted, so that
// these commands don't get forwarded on the internal command topic
// - instead, the second consumer takes over at the offset of the first command (position must have been committed
// when closing the first consumer) and processes and forwards all commands starting with the second command
final int numTestCommands = 10;
final List<KafkaConsumerRecord<String, Buffer>> receivedRecords = new ArrayList<>();
final Promise<Void> firstRecordReceivedPromise = Promise.promise();
final Promise<Void> allRecordsReceivedPromise = Promise.promise();
final List<String> receivedCommandSubjects = new ArrayList<>();
final Handler<KafkaConsumerRecord<String, Buffer>> recordHandler = record -> {
receivedRecords.add(record);
LOG.trace("received {}", record);
receivedCommandSubjects.add(KafkaRecordHelper.getSubject(record.headers()).orElse(""));
if (receivedRecords.size() == 1) {
firstRecordReceivedPromise.complete();
}
if (receivedRecords.size() == numTestCommands) {
allRecordsReceivedPromise.tryComplete();
}
};
final Promise<Void> firstConsumerAllGetAdapterInstanceInvocationsDone = Promise.promise();
final LinkedList<Promise<Void>> firstConsumerGetAdapterInstancePromisesQueue = new LinkedList<>();
// don't let getting the target adapter instance finish immediately
final Supplier<Future<Void>> firstConsumerGetAdapterInstanceSupplier = () -> {
final Promise<Void> resultPromise = Promise.promise();
firstConsumerGetAdapterInstancePromisesQueue.addFirst(resultPromise);
// don't complete the future for the second command here yet
if (firstConsumerGetAdapterInstancePromisesQueue.size() != 2) {
resultPromise.complete();
}
if (firstConsumerGetAdapterInstancePromisesQueue.size() == numTestCommands) {
firstConsumerAllGetAdapterInstanceInvocationsDone.complete();
}
return resultPromise.future();
};
final AtomicReference<KafkaBasedCommandConsumerFactoryImpl> consumerFactory1Ref = new AtomicReference<>();
final Context vertxContext = vertx.getOrCreateContext();
vertxContext.runOnContext(v0 -> {
final HonoKafkaConsumer internalConsumer = getInternalCommandConsumer(recordHandler);
final KafkaBasedCommandConsumerFactoryImpl consumerFactory1 = getKafkaBasedCommandConsumerFactory(firstConsumerGetAdapterInstanceSupplier, tenantId);
consumerFactory1Ref.set(consumerFactory1);
CompositeFuture.join(internalConsumer.start(), consumerFactory1.start()).compose(f -> createCommandConsumer(tenantId, consumerFactory1)).onComplete(setup.succeedingThenComplete());
});
assertThat(setup.awaitCompletion(IntegrationTestSupport.getTestSetupTimeout(), TimeUnit.SECONDS)).isTrue();
if (setup.failed()) {
ctx.failNow(setup.causeOfFailure());
return;
}
LOG.debug("command consumer started");
final List<String> sentCommandSubjects = new ArrayList<>();
IntStream.range(0, numTestCommands).forEach(i -> {
final String subject = "cmd_" + i;
sentCommandSubjects.add(subject);
sendOneWayCommand(tenantId, "myDeviceId", subject);
});
final AtomicInteger secondConsumerGetAdapterInstanceInvocations = new AtomicInteger();
// wait for first record on internal topic to have been received ...
CompositeFuture.join(firstConsumerAllGetAdapterInstanceInvocationsDone.future(), firstRecordReceivedPromise.future()).compose(v -> {
// ... and wait some more, making sure that the offset of the first record has been committed
final Promise<Void> delayPromise = Promise.promise();
vertx.setTimer(500, tid -> delayPromise.complete());
return delayPromise.future();
}).onComplete(v -> {
LOG.info("stopping first consumer factory");
consumerFactory1Ref.get().stop().onComplete(ctx.succeeding(ar -> {
LOG.info("factory stopped");
// no delay on getting the target adapter instance added here
final KafkaBasedCommandConsumerFactoryImpl consumerFactory2 = getKafkaBasedCommandConsumerFactory(() -> {
secondConsumerGetAdapterInstanceInvocations.incrementAndGet();
return Future.succeededFuture();
}, tenantId);
consumerFactory2.start().onComplete(ctx.succeeding(ar2 -> {
LOG.info("creating command consumer in new consumer factory");
createCommandConsumer(tenantId, consumerFactory2).onComplete(ctx.succeeding(ar3 -> {
LOG.debug("consumer created");
firstConsumerGetAdapterInstancePromisesQueue.forEach(Promise::tryComplete);
}));
}));
}));
});
final long timerId = vertx.setTimer(8000, tid -> {
LOG.info("received records:\n{}", receivedRecords.stream().map(Object::toString).collect(Collectors.joining(",\n")));
allRecordsReceivedPromise.tryFail(String.format("only received %d out of %d expected messages after 8s", receivedRecords.size(), numTestCommands));
});
allRecordsReceivedPromise.future().onComplete(ctx.succeeding(v -> {
vertx.cancelTimer(timerId);
ctx.verify(() -> {
assertThat(receivedCommandSubjects).isEqualTo(sentCommandSubjects);
// all but the first command should have been processed by the second consumer
assertThat(secondConsumerGetAdapterInstanceInvocations.get()).isEqualTo(numTestCommands - 1);
});
ctx.completeNow();
}));
}
use of io.vertx.kafka.client.consumer.KafkaConsumerRecord in project hono by eclipse.
the class KafkaBasedMappingAndDelegatingCommandHandlerTest method testIncomingCommandOrderIsPreservedWhenDelegating.
/**
* Verifies the behaviour of the
* {@link KafkaBasedMappingAndDelegatingCommandHandler#mapAndDelegateIncomingCommandMessage(KafkaConsumerRecord)}
* method in a scenario where the mapping operation for one command completes earlier than for a previously received
* command. The order in which commands are then delegated to the target adapter instance has to be the same
* as the order in which commands were received.
*
* @param ctx The vert.x test context
*/
@Test
public void testIncomingCommandOrderIsPreservedWhenDelegating(final VertxTestContext ctx) {
final String deviceId1 = "device1";
final String deviceId2 = "device2";
final String deviceId3 = "device3";
final String deviceId4 = "device4";
// GIVEN valid command records
final KafkaConsumerRecord<String, Buffer> commandRecord1 = getCommandRecord(tenantId, deviceId1, "subject1", 0, 1);
final KafkaConsumerRecord<String, Buffer> commandRecord2 = getCommandRecord(tenantId, deviceId2, "subject2", 0, 2);
final KafkaConsumerRecord<String, Buffer> commandRecord3 = getCommandRecord(tenantId, deviceId3, "subject3", 0, 3);
final KafkaConsumerRecord<String, Buffer> commandRecord4 = getCommandRecord(tenantId, deviceId4, "subject4", 0, 4);
// WHEN getting the target adapter instances for the commands results in different delays for each command
// so that the invocations are completed with the order: commandRecord3, commandRecord2, commandRecord1, commandRecord4
final Promise<JsonObject> resultForCommand1 = Promise.promise();
when(commandTargetMapper.getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId1), any())).thenReturn(resultForCommand1.future());
final Promise<JsonObject> resultForCommand2 = Promise.promise();
when(commandTargetMapper.getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId2), any())).thenReturn(resultForCommand2.future());
final Promise<JsonObject> resultForCommand3 = Promise.promise();
when(commandTargetMapper.getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId3), any())).thenReturn(resultForCommand3.future());
doAnswer(invocation -> {
resultForCommand3.complete(createTargetAdapterInstanceJson(deviceId3, adapterInstanceId));
resultForCommand2.complete(createTargetAdapterInstanceJson(deviceId2, adapterInstanceId));
resultForCommand1.complete(createTargetAdapterInstanceJson(deviceId1, adapterInstanceId));
return Future.succeededFuture(createTargetAdapterInstanceJson(deviceId4, adapterInstanceId));
}).when(commandTargetMapper).getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId4), any());
// WHEN mapping and delegating the commands
final Future<Void> cmd1Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord1);
final Future<Void> cmd2Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord2);
final Future<Void> cmd3Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord3);
final Future<Void> cmd4Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord4);
// THEN the messages are delegated in the original order
CompositeFuture.all(cmd1Future, cmd2Future, cmd3Future, cmd4Future).onComplete(ctx.succeeding(r -> {
ctx.verify(() -> {
final ArgumentCaptor<CommandContext> commandContextCaptor = ArgumentCaptor.forClass(CommandContext.class);
verify(internalCommandSender, times(4)).sendCommand(commandContextCaptor.capture(), anyString());
final List<CommandContext> capturedCommandContexts = commandContextCaptor.getAllValues();
assertThat(capturedCommandContexts.get(0).getCommand().getDeviceId()).isEqualTo(deviceId1);
assertThat(capturedCommandContexts.get(1).getCommand().getDeviceId()).isEqualTo(deviceId2);
assertThat(capturedCommandContexts.get(2).getCommand().getDeviceId()).isEqualTo(deviceId3);
assertThat(capturedCommandContexts.get(3).getCommand().getDeviceId()).isEqualTo(deviceId4);
});
ctx.completeNow();
}));
}
Aggregations