use of org.apache.hudi.connect.transaction.ConnectTransactionCoordinator in project hudi by apache.
the class HoodieSinkTask method bootstrap.
private void bootstrap(Collection<TopicPartition> partitions) {
LOG.info(String.format("Bootstrap task for connector %s with id %s with assignments %s part %s", connectorName, taskId, context.assignment(), partitions));
for (TopicPartition partition : partitions) {
try {
// If the partition is 0, instantiate the Leader
if (partition.partition() == ConnectTransactionCoordinator.COORDINATOR_KAFKA_PARTITION) {
ConnectTransactionCoordinator coordinator = new ConnectTransactionCoordinator(connectConfigs, partition, controlKafkaClient);
coordinator.start();
transactionCoordinators.put(partition, coordinator);
}
ConnectTransactionParticipant worker = new ConnectTransactionParticipant(connectConfigs, partition, controlKafkaClient, context);
transactionParticipants.put(partition, worker);
worker.start();
} catch (HoodieException exception) {
LOG.error(String.format("Fatal error initializing task %s for partition %s", taskId, partition.partition()), exception);
}
}
}
use of org.apache.hudi.connect.transaction.ConnectTransactionCoordinator in project hudi by apache.
the class TestConnectTransactionCoordinator method testSingleCommitScenario.
@ParameterizedTest
@EnumSource(value = MockParticipant.TestScenarios.class)
public void testSingleCommitScenario(MockParticipant.TestScenarios scenario) throws InterruptedException {
kafkaControlAgent = new MockKafkaControlAgent();
participant = new MockParticipant(kafkaControlAgent, latch, scenario, MAX_COMMIT_ROUNDS);
participant.start();
KafkaConnectConfigs.Builder configBuilder = KafkaConnectConfigs.newBuilder().withCommitIntervalSecs(1L).withCoordinatorWriteTimeoutSecs(1L);
if (scenario.equals(MockParticipant.TestScenarios.SUBSET_WRITE_STATUS_FAILED)) {
configBuilder.withAllowCommitOnErrors(false);
}
configs = configBuilder.build();
// Test the coordinator using the mock participant
TransactionCoordinator coordinator = new ConnectTransactionCoordinator(configs, new TopicPartition(TOPIC_NAME, 0), kafkaControlAgent, transactionServices, (bootstrapServers, topicName) -> TOTAL_KAFKA_PARTITIONS);
coordinator.start();
latch.await(TEST_TIMEOUT_SECS, TimeUnit.SECONDS);
if (latch.getCount() > 0) {
throw new HoodieException("Test timedout resulting in failure");
}
coordinator.stop();
participant.stop();
}
Aggregations