Examples with ControlMessage org.apache.hudi.connect.ControlMessage used on opensource projects

Example 1 with ControlMessage

use of org.apache.hudi.connect.ControlMessage in project hudi by apache.

the class KafkaConnectControlAgent method start.

private void start() {
    Properties props = new Properties();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    // Todo fetch the worker id or name instead of a uuid.
    props.put(ConsumerConfig.GROUP_ID_CONFIG, "hudi-control-group" + UUID.randomUUID().toString());
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class);
    // Since we are using Kafka Control Topic as a RPC like interface,
    // we want consumers to only process messages that are sent after they come online
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
    consumer = new KafkaConsumer<>(props, new StringDeserializer(), new ByteArrayDeserializer());
    consumer.subscribe(Collections.singletonList(controlTopicName));
    executorService.submit(() -> {
        while (true) {
            ConsumerRecords<String, byte[]> records;
            records = consumer.poll(Duration.ofMillis(KAFKA_POLL_TIMEOUT_MS));
            for (ConsumerRecord<String, byte[]> record : records) {
                try {
                    LOG.debug(String.format("Kafka consumerGroupId = %s topic = %s, partition = %s, offset = %s, customer = %s, country = %s", "", record.topic(), record.partition(), record.offset(), record.key(), record.value()));
                    ControlMessage message = ControlMessage.parseFrom(record.value());
                    String senderTopic = message.getTopicName();
                    if (message.getReceiverType().equals(ControlMessage.EntityType.PARTICIPANT)) {
                        if (partitionWorkers.containsKey(senderTopic)) {
                            for (TransactionParticipant partitionWorker : partitionWorkers.get(senderTopic)) {
                                partitionWorker.processControlEvent(message);
                            }
                        } else {
                            LOG.warn(String.format("Failed to send message for unregistered participants for topic %s", senderTopic));
                        }
                    } else if (message.getReceiverType().equals(ControlMessage.EntityType.COORDINATOR)) {
                        if (topicCoordinators.containsKey(senderTopic)) {
                            topicCoordinators.get(senderTopic).processControlEvent(message);
                        }
                    } else {
                        LOG.warn(String.format("Sender type of Control Message unknown %s", message.getSenderType().name()));
                    }
                } catch (Exception e) {
                    LOG.error(String.format("Fatal error while consuming a kafka record for topic = %s partition = %s", record.topic(), record.partition()), e);
                }
            }
            try {
                consumer.commitSync();
            } catch (CommitFailedException exception) {
                LOG.error("Fatal error while committing kafka control topic");
            }
        }
    });
}

Example 2 with ControlMessage

use of org.apache.hudi.connect.ControlMessage in project hudi by apache.

the class ConnectTransactionParticipant method handleEndCommit.

private void handleEndCommit(ControlMessage message) {
    if (ongoingTransactionInfo == null) {
        LOG.warn(String.format("END_COMMIT %s is received while we were NOT in active transaction", message.getCommitTime()));
        return;
    } else if (!ongoingTransactionInfo.getCommitTime().equals(message.getCommitTime())) {
        LOG.error(String.format("Fatal error received END_COMMIT with commit time %s while local transaction commit time %s", message.getCommitTime(), ongoingTransactionInfo.getCommitTime()));
        // Recovery: A new END_COMMIT from leader caused interruption to an existing transaction,
        // explicitly reset Kafka commit offset to ensure no data loss
        cleanupOngoingTransaction();
        syncKafkaOffsetWithLeader(message);
        return;
    }
    context.pause(partition);
    ongoingTransactionInfo.commitInitiated();
    // send Writer Status Message and wait for ACK_COMMIT in async fashion
    try {
        // sendWriterStatus
        List<WriteStatus> writeStatuses = ongoingTransactionInfo.getWriter().close();
        ControlMessage writeStatusEvent = ControlMessage.newBuilder().setProtocolVersion(KafkaConnectConfigs.CURRENT_PROTOCOL_VERSION).setType(ControlMessage.EventType.WRITE_STATUS).setTopicName(partition.topic()).setSenderType(ControlMessage.EntityType.PARTICIPANT).setSenderPartition(partition.partition()).setReceiverType(ControlMessage.EntityType.COORDINATOR).setReceiverPartition(ConnectTransactionCoordinator.COORDINATOR_KAFKA_PARTITION).setCommitTime(ongoingTransactionInfo.getCommitTime()).setParticipantInfo(ControlMessage.ParticipantInfo.newBuilder().setWriteStatus(KafkaConnectUtils.buildWriteStatuses(writeStatuses)).setKafkaOffset(ongoingTransactionInfo.getExpectedKafkaOffset()).build()).build();
        kafkaControlAgent.publishMessage(writeStatusEvent);
    } catch (Exception exception) {
        LOG.error(String.format("Error writing records and ending commit %s for partition %s", message.getCommitTime(), partition.partition()), exception);
        throw new HoodieIOException(String.format("Error writing records and ending commit %s for partition %s", message.getCommitTime(), partition.partition()), new IOException(exception));
    }
}