Examples with KafkaConsumerRecord io.vertx.kafka.client.consumer.KafkaConsumerRecord used on opensource projects

Example 26 with KafkaConsumerRecord

use of io.vertx.kafka.client.consumer.KafkaConsumerRecord in project hono by eclipse.

the class KafkaBasedCommandConsumerFactoryImplIT method testCommandsGetForwardedIfOneConsumerInstanceGetsClosed.

/**
 * Verifies that records, published on the tenant-specific Kafka command topic, get received
 * and forwarded by consumers created by factory instances even if one factory and its contained
 * consumer gets closed in the middle of processing some of the commands.
 *
 * @param ctx The vert.x test context.
 * @throws InterruptedException if test execution gets interrupted.
 */
@Test
@Timeout(value = 10, timeUnit = TimeUnit.SECONDS)
public void testCommandsGetForwardedIfOneConsumerInstanceGetsClosed(final VertxTestContext ctx) throws InterruptedException {
    final String tenantId = "tenant_" + UUID.randomUUID();
    final VertxTestContext setup = new VertxTestContext();
    // Scenario to test:
    // - first command gets sent, forwarded and received without any imposed delay
    // - second command gets sent, received by the factory consumer instance; processing gets blocked
    // while trying to get the target adapter instance
    // - for the rest of the commands, retrieval of the target adapter instance is successful, but they won't
    // get forwarded until processing of the second command is finished
    // - now the factory consumer gets closed and a new factory/consumer gets started; at that point
    // also the processing of the second command gets finished
    // 
    // Expected outcome:
    // - processing of the second command and all following commands by the first consumer gets aborted, so that
    // these commands don't get forwarded on the internal command topic
    // - instead, the second consumer takes over at the offset of the first command (position must have been committed
    // when closing the first consumer) and processes and forwards all commands starting with the second command
    final int numTestCommands = 10;
    final List<KafkaConsumerRecord<String, Buffer>> receivedRecords = new ArrayList<>();
    final Promise<Void> firstRecordReceivedPromise = Promise.promise();
    final Promise<Void> allRecordsReceivedPromise = Promise.promise();
    final List<String> receivedCommandSubjects = new ArrayList<>();
    final Handler<KafkaConsumerRecord<String, Buffer>> recordHandler = record -> {
        receivedRecords.add(record);
        LOG.trace("received {}", record);
        receivedCommandSubjects.add(KafkaRecordHelper.getSubject(record.headers()).orElse(""));
        if (receivedRecords.size() == 1) {
            firstRecordReceivedPromise.complete();
        }
        if (receivedRecords.size() == numTestCommands) {
            allRecordsReceivedPromise.tryComplete();
        }
    };
    final Promise<Void> firstConsumerAllGetAdapterInstanceInvocationsDone = Promise.promise();
    final LinkedList<Promise<Void>> firstConsumerGetAdapterInstancePromisesQueue = new LinkedList<>();
    // don't let getting the target adapter instance finish immediately
    final Supplier<Future<Void>> firstConsumerGetAdapterInstanceSupplier = () -> {
        final Promise<Void> resultPromise = Promise.promise();
        firstConsumerGetAdapterInstancePromisesQueue.addFirst(resultPromise);
        // don't complete the future for the second command here yet
        if (firstConsumerGetAdapterInstancePromisesQueue.size() != 2) {
            resultPromise.complete();
        }
        if (firstConsumerGetAdapterInstancePromisesQueue.size() == numTestCommands) {
            firstConsumerAllGetAdapterInstanceInvocationsDone.complete();
        }
        return resultPromise.future();
    };
    final AtomicReference<KafkaBasedCommandConsumerFactoryImpl> consumerFactory1Ref = new AtomicReference<>();
    final Context vertxContext = vertx.getOrCreateContext();
    vertxContext.runOnContext(v0 -> {
        final HonoKafkaConsumer internalConsumer = getInternalCommandConsumer(recordHandler);
        final KafkaBasedCommandConsumerFactoryImpl consumerFactory1 = getKafkaBasedCommandConsumerFactory(firstConsumerGetAdapterInstanceSupplier, tenantId);
        consumerFactory1Ref.set(consumerFactory1);
        CompositeFuture.join(internalConsumer.start(), consumerFactory1.start()).compose(f -> createCommandConsumer(tenantId, consumerFactory1)).onComplete(setup.succeedingThenComplete());
    });
    assertThat(setup.awaitCompletion(IntegrationTestSupport.getTestSetupTimeout(), TimeUnit.SECONDS)).isTrue();
    if (setup.failed()) {
        ctx.failNow(setup.causeOfFailure());
        return;
    }
    LOG.debug("command consumer started");
    final List<String> sentCommandSubjects = new ArrayList<>();
    IntStream.range(0, numTestCommands).forEach(i -> {
        final String subject = "cmd_" + i;
        sentCommandSubjects.add(subject);
        sendOneWayCommand(tenantId, "myDeviceId", subject);
    });
    final AtomicInteger secondConsumerGetAdapterInstanceInvocations = new AtomicInteger();
    // wait for first record on internal topic to have been received ...
    CompositeFuture.join(firstConsumerAllGetAdapterInstanceInvocationsDone.future(), firstRecordReceivedPromise.future()).compose(v -> {
        // ... and wait some more, making sure that the offset of the first record has been committed
        final Promise<Void> delayPromise = Promise.promise();
        vertx.setTimer(500, tid -> delayPromise.complete());
        return delayPromise.future();
    }).onComplete(v -> {
        LOG.info("stopping first consumer factory");
        consumerFactory1Ref.get().stop().onComplete(ctx.succeeding(ar -> {
            LOG.info("factory stopped");
            // no delay on getting the target adapter instance added here
            final KafkaBasedCommandConsumerFactoryImpl consumerFactory2 = getKafkaBasedCommandConsumerFactory(() -> {
                secondConsumerGetAdapterInstanceInvocations.incrementAndGet();
                return Future.succeededFuture();
            }, tenantId);
            consumerFactory2.start().onComplete(ctx.succeeding(ar2 -> {
                LOG.info("creating command consumer in new consumer factory");
                createCommandConsumer(tenantId, consumerFactory2).onComplete(ctx.succeeding(ar3 -> {
                    LOG.debug("consumer created");
                    firstConsumerGetAdapterInstancePromisesQueue.forEach(Promise::tryComplete);
                }));
            }));
        }));
    });
    final long timerId = vertx.setTimer(8000, tid -> {
        LOG.info("received records:\n{}", receivedRecords.stream().map(Object::toString).collect(Collectors.joining(",\n")));
        allRecordsReceivedPromise.tryFail(String.format("only received %d out of %d expected messages after 8s", receivedRecords.size(), numTestCommands));
    });
    allRecordsReceivedPromise.future().onComplete(ctx.succeeding(v -> {
        vertx.cancelTimer(timerId);
        ctx.verify(() -> {
            assertThat(receivedCommandSubjects).isEqualTo(sentCommandSubjects);
            // all but the first command should have been processed by the second consumer
            assertThat(secondConsumerGetAdapterInstanceInvocations.get()).isEqualTo(numTestCommands - 1);
        });
        ctx.completeNow();
    }));
}

Example 27 with KafkaConsumerRecord

use of io.vertx.kafka.client.consumer.KafkaConsumerRecord in project hono by eclipse.

the class KafkaBasedMappingAndDelegatingCommandHandlerTest method testIncomingCommandOrderIsPreservedWhenDelegating.

/**
 * Verifies the behaviour of the
 * {@link KafkaBasedMappingAndDelegatingCommandHandler#mapAndDelegateIncomingCommandMessage(KafkaConsumerRecord)}
 * method in a scenario where the mapping operation for one command completes earlier than for a previously received
 * command. The order in which commands are then delegated to the target adapter instance has to be the same
 * as the order in which commands were received.
 *
 * @param ctx The vert.x test context
 */
@Test
public void testIncomingCommandOrderIsPreservedWhenDelegating(final VertxTestContext ctx) {
    final String deviceId1 = "device1";
    final String deviceId2 = "device2";
    final String deviceId3 = "device3";
    final String deviceId4 = "device4";
    // GIVEN valid command records
    final KafkaConsumerRecord<String, Buffer> commandRecord1 = getCommandRecord(tenantId, deviceId1, "subject1", 0, 1);
    final KafkaConsumerRecord<String, Buffer> commandRecord2 = getCommandRecord(tenantId, deviceId2, "subject2", 0, 2);
    final KafkaConsumerRecord<String, Buffer> commandRecord3 = getCommandRecord(tenantId, deviceId3, "subject3", 0, 3);
    final KafkaConsumerRecord<String, Buffer> commandRecord4 = getCommandRecord(tenantId, deviceId4, "subject4", 0, 4);
    // WHEN getting the target adapter instances for the commands results in different delays for each command
    // so that the invocations are completed with the order: commandRecord3, commandRecord2, commandRecord1, commandRecord4
    final Promise<JsonObject> resultForCommand1 = Promise.promise();
    when(commandTargetMapper.getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId1), any())).thenReturn(resultForCommand1.future());
    final Promise<JsonObject> resultForCommand2 = Promise.promise();
    when(commandTargetMapper.getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId2), any())).thenReturn(resultForCommand2.future());
    final Promise<JsonObject> resultForCommand3 = Promise.promise();
    when(commandTargetMapper.getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId3), any())).thenReturn(resultForCommand3.future());
    doAnswer(invocation -> {
        resultForCommand3.complete(createTargetAdapterInstanceJson(deviceId3, adapterInstanceId));
        resultForCommand2.complete(createTargetAdapterInstanceJson(deviceId2, adapterInstanceId));
        resultForCommand1.complete(createTargetAdapterInstanceJson(deviceId1, adapterInstanceId));
        return Future.succeededFuture(createTargetAdapterInstanceJson(deviceId4, adapterInstanceId));
    }).when(commandTargetMapper).getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId4), any());
    // WHEN mapping and delegating the commands
    final Future<Void> cmd1Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord1);
    final Future<Void> cmd2Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord2);
    final Future<Void> cmd3Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord3);
    final Future<Void> cmd4Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord4);
    // THEN the messages are delegated in the original order
    CompositeFuture.all(cmd1Future, cmd2Future, cmd3Future, cmd4Future).onComplete(ctx.succeeding(r -> {
        ctx.verify(() -> {
            final ArgumentCaptor<CommandContext> commandContextCaptor = ArgumentCaptor.forClass(CommandContext.class);
            verify(internalCommandSender, times(4)).sendCommand(commandContextCaptor.capture(), anyString());
            final List<CommandContext> capturedCommandContexts = commandContextCaptor.getAllValues();
            assertThat(capturedCommandContexts.get(0).getCommand().getDeviceId()).isEqualTo(deviceId1);
            assertThat(capturedCommandContexts.get(1).getCommand().getDeviceId()).isEqualTo(deviceId2);
            assertThat(capturedCommandContexts.get(2).getCommand().getDeviceId()).isEqualTo(deviceId3);
            assertThat(capturedCommandContexts.get(3).getCommand().getDeviceId()).isEqualTo(deviceId4);
        });
        ctx.completeNow();
    }));
}