Examples with KafkaConsumerRecord io.vertx.kafka.client.consumer.KafkaConsumerRecord used on opensource projects

Example 11 with KafkaConsumerRecord

use of io.vertx.kafka.client.consumer.KafkaConsumerRecord in project hono by eclipse.

the class KafkaBasedMappingAndDelegatingCommandHandlerTest method testCommandDelegationOrderWithMappingFailedForFirstEntry.

/**
 * Verifies the behaviour of the
 * {@link KafkaBasedMappingAndDelegatingCommandHandler#mapAndDelegateIncomingCommandMessage(KafkaConsumerRecord)}
 * method in a scenario where the rather long-running processing of a command delays subsequent, already mapped
 * commands from getting delegated to the target adapter instance. After the processing of the first command finally
 * resulted in an error, the subsequent commands shall get delegated in the correct order.
 *
 * @param ctx The vert.x test context
 */
@Test
public void testCommandDelegationOrderWithMappingFailedForFirstEntry(final VertxTestContext ctx) {
    final String deviceId1 = "device1";
    final String deviceId2 = "device2";
    final String deviceId3 = "device3";
    final String deviceId4 = "device4";
    // GIVEN valid command records
    final KafkaConsumerRecord<String, Buffer> commandRecord1 = getCommandRecord(tenantId, deviceId1, "subject1", 0, 1);
    final KafkaConsumerRecord<String, Buffer> commandRecord2 = getCommandRecord(tenantId, deviceId2, "subject2", 0, 2);
    final KafkaConsumerRecord<String, Buffer> commandRecord3 = getCommandRecord(tenantId, deviceId3, "subject3", 0, 3);
    final KafkaConsumerRecord<String, Buffer> commandRecord4 = getCommandRecord(tenantId, deviceId4, "subject4", 0, 4);
    // WHEN getting the target adapter instances for the commands results in different delays for each command
    // so that the invocations are completed with the order: commandRecord3, commandRecord2, commandRecord1 (failed), commandRecord4
    // with command 1 getting failed
    final Promise<JsonObject> resultForCommand1 = Promise.promise();
    when(commandTargetMapper.getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId1), any())).thenReturn(resultForCommand1.future());
    final Promise<JsonObject> resultForCommand2 = Promise.promise();
    when(commandTargetMapper.getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId2), any())).thenReturn(resultForCommand2.future());
    final Promise<JsonObject> resultForCommand3 = Promise.promise();
    when(commandTargetMapper.getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId3), any())).thenReturn(resultForCommand3.future());
    doAnswer(invocation -> {
        resultForCommand3.complete(createTargetAdapterInstanceJson(deviceId3, adapterInstanceId));
        resultForCommand2.complete(createTargetAdapterInstanceJson(deviceId2, adapterInstanceId));
        resultForCommand1.fail("mapping of command 1 failed for some reason");
        return Future.succeededFuture(createTargetAdapterInstanceJson(deviceId4, adapterInstanceId));
    }).when(commandTargetMapper).getTargetGatewayAndAdapterInstance(eq(tenantId), eq(deviceId4), any());
    // WHEN mapping and delegating the commands
    final Future<Void> cmd1Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord1);
    final Future<Void> cmd2Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord2);
    final Future<Void> cmd3Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord3);
    final Future<Void> cmd4Future = cmdHandler.mapAndDelegateIncomingCommandMessage(commandRecord4);
    // THEN the messages are delegated in the original order, with command 1 left out because it timed out
    CompositeFuture.all(cmd2Future, cmd3Future, cmd4Future).onComplete(ctx.succeeding(r -> {
        ctx.verify(() -> {
            assertThat(cmd1Future.failed()).isTrue();
            final ArgumentCaptor<CommandContext> commandContextCaptor = ArgumentCaptor.forClass(CommandContext.class);
            verify(internalCommandSender, times(3)).sendCommand(commandContextCaptor.capture(), anyString());
            final List<CommandContext> capturedCommandContexts = commandContextCaptor.getAllValues();
            assertThat(capturedCommandContexts.get(0).getCommand().getDeviceId()).isEqualTo(deviceId2);
            assertThat(capturedCommandContexts.get(1).getCommand().getDeviceId()).isEqualTo(deviceId3);
            assertThat(capturedCommandContexts.get(2).getCommand().getDeviceId()).isEqualTo(deviceId4);
        });
        ctx.completeNow();
    }));
}

Example 12 with KafkaConsumerRecord

use of io.vertx.kafka.client.consumer.KafkaConsumerRecord in project hono by eclipse.

the class KafkaBasedInternalCommandConsumerTest method testHandleCommandMessageSendErrorResponse.

/**
 * Verifies that an error response is sent to the application if the tenant of the target device
 * is unknown or cannot be retrieved.
 */
@ParameterizedTest
@ValueSource(ints = { HttpURLConnection.HTTP_NOT_FOUND, HttpURLConnection.HTTP_UNAVAILABLE })
void testHandleCommandMessageSendErrorResponse(final int tenantServiceErrorCode) {
    final String tenantId = "myTenant";
    final String deviceId = "4711";
    final String subject = "subject";
    final Handler<CommandContext> commandHandler = VertxMockSupport.mockHandler();
    commandHandlers.putCommandHandler(tenantId, deviceId, null, commandHandler, context);
    when(tenantClient.get(eq("myTenant"), any())).thenReturn(Future.failedFuture(ServiceInvocationException.create(tenantServiceErrorCode)));
    final KafkaConsumerRecord<String, Buffer> commandRecord = getCommandRecord(deviceId, getHeaders(tenantId, deviceId, subject, 0L));
    internalCommandConsumer.handleCommandMessage(commandRecord);
    verify(commandHandler, never()).handle(any(KafkaBasedCommandContext.class));
    verify(commandResponseSender).sendCommandResponse(argThat(t -> t.getTenantId().equals("myTenant")), argThat(r -> r.getDeviceId().equals("4711")), argThat(cr -> cr.getStatus() == tenantServiceErrorCode), any());
}

Example 13 with KafkaConsumerRecord

use of io.vertx.kafka.client.consumer.KafkaConsumerRecord in project hono by eclipse.

the class KafkaBasedCommand method from.

private static KafkaBasedCommand from(final KafkaConsumerRecord<String, Buffer> record, final String tenantId) {
    final String deviceId = KafkaRecordHelper.getDeviceId(record.headers()).filter(id -> !id.isEmpty()).orElseThrow(() -> new IllegalArgumentException("device identifier is not set"));
    if (!deviceId.equals(record.key())) {
        throw new IllegalArgumentException("device identifier not set as record key");
    }
    final StringJoiner validationErrorJoiner = new StringJoiner(", ");
    final String subject = KafkaRecordHelper.getSubject(record.headers()).orElseGet(() -> {
        validationErrorJoiner.add("subject not set");
        return null;
    });
    final String contentType = KafkaRecordHelper.getContentType(record.headers()).orElse(null);
    final boolean responseRequired = KafkaRecordHelper.isResponseRequired(record.headers());
    final String correlationId = KafkaRecordHelper.getCorrelationId(record.headers()).filter(id -> !id.isEmpty()).orElseGet(() -> {
        if (responseRequired) {
            validationErrorJoiner.add("correlation-id is not set");
        }
        return null;
    });
    return new KafkaBasedCommand(validationErrorJoiner.length() > 0 ? Optional.of(validationErrorJoiner.toString()) : Optional.empty(), record, tenantId, deviceId, correlationId, subject, contentType, responseRequired);
}

Example 14 with KafkaConsumerRecord

use of io.vertx.kafka.client.consumer.KafkaConsumerRecord in project hono by eclipse.

the class KafkaBasedInternalCommandConsumer method handleCommandMessage.

void handleCommandMessage(final KafkaConsumerRecord<String, Buffer> record) {
    // get partition/offset of the command record - related to the tenant-based topic the command was originally received in
    final Integer commandPartition = KafkaRecordHelper.getOriginalPartitionHeader(record.headers()).orElse(null);
    final Long commandOffset = KafkaRecordHelper.getOriginalOffsetHeader(record.headers()).orElse(null);
    if (commandPartition == null || commandOffset == null) {
        LOG.warn("command record is invalid - missing required original partition/offset headers");
        return;
    }
    final KafkaBasedCommand command;
    try {
        command = KafkaBasedCommand.fromRoutedCommandRecord(record);
    } catch (final IllegalArgumentException e) {
        LOG.warn("command record is invalid [tenant-id: {}, device-id: {}]", KafkaRecordHelper.getTenantId(record.headers()).orElse(null), KafkaRecordHelper.getDeviceId(record.headers()).orElse(null), e);
        return;
    }
    // check whether command has already been received and handled;
    // partition index and offset here are related to the *tenant-based* topic the command was originally received in
    // therefore they are stored in a map with the tenant as key
    final Map<Integer, Long> lastHandledPartitionOffsets = lastHandledPartitionOffsetsPerTenant.computeIfAbsent(command.getTenant(), k -> new HashMap<>());
    final Long lastHandledOffset = lastHandledPartitionOffsets.get(commandPartition);
    if (lastHandledOffset != null && commandOffset <= lastHandledOffset) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("ignoring command - record partition offset {} <= last handled offset {} [{}]", commandOffset, lastHandledOffset, command);
        }
    } else {
        lastHandledPartitionOffsets.put(commandPartition, commandOffset);
        final CommandHandlerWrapper commandHandler = commandHandlers.getCommandHandler(command.getTenant(), command.getGatewayOrDeviceId());
        if (commandHandler != null && commandHandler.getGatewayId() != null) {
            // Gateway information set in command handler means a gateway has subscribed for commands for a specific device.
            // This information isn't getting set in the record (by the Command Router) and therefore has to be adopted manually here.
            command.setGatewayId(commandHandler.getGatewayId());
        }
        final SpanContext spanContext = KafkaTracingHelper.extractSpanContext(tracer, record);
        final SpanContext followsFromSpanContext = commandHandler != null ? commandHandler.getConsumerCreationSpanContext() : null;
        final Span currentSpan = CommandContext.createSpan(tracer, command, spanContext, followsFromSpanContext, getClass().getSimpleName());
        currentSpan.setTag(MessageHelper.APP_PROPERTY_ADAPTER_INSTANCE_ID, adapterInstanceId);
        KafkaTracingHelper.TAG_OFFSET.set(currentSpan, record.offset());
        final var commandContext = new KafkaBasedCommandContext(command, commandResponseSender, currentSpan);
        tenantClient.get(command.getTenant(), spanContext).onFailure(t -> {
            if (ServiceInvocationException.extractStatusCode(t) == HttpURLConnection.HTTP_NOT_FOUND) {
                commandContext.reject(new TenantDisabledOrNotRegisteredException(command.getTenant(), HttpURLConnection.HTTP_NOT_FOUND));
            } else {
                commandContext.release(new ServerErrorException(command.getTenant(), HttpURLConnection.HTTP_UNAVAILABLE, "error retrieving tenant configuration", t));
            }
        }).onSuccess(tenantConfig -> {
            commandContext.put(CommandContext.KEY_TENANT_CONFIG, tenantConfig);
            if (commandHandler != null) {
                LOG.trace("using [{}] for received command [{}]", commandHandler, command);
                // command.isValid() check not done here - it is to be done in the command handler
                commandHandler.handleCommand(commandContext);
            } else {
                LOG.info("no command handler found for command [{}]", command);
                commandContext.release(new NoConsumerException("no command handler found for command"));
            }
        });
    }
}

Example 15 with KafkaConsumerRecord

use of io.vertx.kafka.client.consumer.KafkaConsumerRecord in project hono by eclipse.

the class KafkaApplicationClientImpl method createKafkaBasedDownstreamMessageConsumer.

private Future<MessageConsumer> createKafkaBasedDownstreamMessageConsumer(final String tenantId, final HonoTopic.Type type, final Handler<DownstreamMessage<KafkaMessageContext>> messageHandler) {
    Objects.requireNonNull(tenantId);
    Objects.requireNonNull(type);
    Objects.requireNonNull(messageHandler);
    final String topic = new HonoTopic(type, tenantId).toString();
    final Handler<KafkaConsumerRecord<String, Buffer>> recordHandler = record -> {
        messageHandler.handle(new KafkaDownstreamMessage(record));
    };
    final HonoKafkaConsumer consumer = new HonoKafkaConsumer(vertx, Set.of(topic), recordHandler, consumerConfig.getConsumerConfig(type.toString()));
    consumer.setPollTimeout(Duration.ofMillis(consumerConfig.getPollTimeout()));
    Optional.ofNullable(kafkaConsumerSupplier).ifPresent(consumer::setKafkaConsumerSupplier);
    return consumer.start().map(v -> (MessageConsumer) new MessageConsumer() {

        @Override
        public Future<Void> close() {
            return consumer.stop();
        }
    }).onSuccess(consumersToCloseOnStop::add);
}