Examples with Checkpoint org.apache.kafka.connect.mirror.Checkpoint used on opensource projects

Example 1 with Checkpoint

use of org.apache.kafka.connect.mirror.Checkpoint in project kafka by apache.

the class MirrorConnectorsIntegrationBaseTest method testOffsetSyncsTopicsOnTarget.

@Test
public void testOffsetSyncsTopicsOnTarget() throws Exception {
    // move offset-syncs topics to target
    mm2Props.put(PRIMARY_CLUSTER_ALIAS + "->" + BACKUP_CLUSTER_ALIAS + ".offset-syncs.topic.location", "target");
    // one way replication from primary to backup
    mm2Props.put(BACKUP_CLUSTER_ALIAS + "->" + PRIMARY_CLUSTER_ALIAS + ".enabled", "false");
    mm2Config = new MirrorMakerConfig(mm2Props);
    waitUntilMirrorMakerIsRunning(backup, CONNECTOR_LIST, mm2Config, PRIMARY_CLUSTER_ALIAS, BACKUP_CLUSTER_ALIAS);
    // Ensure the offset syncs topic is created in the target cluster
    waitForTopicCreated(backup.kafka(), "mm2-offset-syncs." + PRIMARY_CLUSTER_ALIAS + ".internal");
    produceMessages(primary, "test-topic-1");
    ReplicationPolicy replicationPolicy = new MirrorClient(mm2Config.clientConfig(BACKUP_CLUSTER_ALIAS)).replicationPolicy();
    String remoteTopic = replicationPolicy.formatRemoteTopic(PRIMARY_CLUSTER_ALIAS, "test-topic-1");
    // Check offsets are pushed to the checkpoint topic
    Consumer<byte[], byte[]> backupConsumer = backup.kafka().createConsumerAndSubscribeTo(Collections.singletonMap("auto.offset.reset", "earliest"), PRIMARY_CLUSTER_ALIAS + ".checkpoints.internal");
    waitForCondition(() -> {
        ConsumerRecords<byte[], byte[]> records = backupConsumer.poll(Duration.ofSeconds(1L));
        for (ConsumerRecord<byte[], byte[]> record : records) {
            Checkpoint checkpoint = Checkpoint.deserializeRecord(record);
            if (remoteTopic.equals(checkpoint.topicPartition().topic())) {
                return true;
            }
        }
        return false;
    }, 30_000, "Unable to find checkpoints for " + PRIMARY_CLUSTER_ALIAS + ".test-topic-1");
    // Ensure no offset-syncs topics have been created on the primary cluster
    Set<String> primaryTopics = primary.kafka().createAdminClient().listTopics().names().get();
    assertFalse(primaryTopics.contains("mm2-offset-syncs." + PRIMARY_CLUSTER_ALIAS + ".internal"));
    assertFalse(primaryTopics.contains("mm2-offset-syncs." + BACKUP_CLUSTER_ALIAS + ".internal"));
}