Examples with SourceRecord org.apache.kafka.connect.source.SourceRecord used on opensource projects

Example 76 with SourceRecord

use of org.apache.kafka.connect.source.SourceRecord in project kafka by apache.

the class SchemaSourceTask method poll.

@Override
public List<SourceRecord> poll() {
    if (count < maxNumMsgs) {
        long sendStartMs = System.currentTimeMillis();
        if (throttler.shouldThrottle(seqno - startingSeqno, sendStartMs)) {
            throttler.throttle();
        }
        Map<String, Long> ccOffset = Collections.singletonMap(SEQNO_FIELD, seqno);
        int partitionVal = (int) (seqno % partitionCount);
        final Struct data;
        final SourceRecord srcRecord;
        if (!multipleSchema || count % 2 == 0) {
            data = new Struct(VALUE_SCHEMA).put("boolean", true).put("int", 12).put("long", 12L).put("float", 12.2f).put("double", 12.2).put("partitioning", partitionVal).put("id", id).put("seqno", seqno);
            srcRecord = new SourceRecord(partition, ccOffset, topic, id, Schema.STRING_SCHEMA, "key", VALUE_SCHEMA, data);
        } else {
            data = new Struct(VALUE_SCHEMA_2).put("boolean", true).put("int", 12).put("long", 12L).put("float", 12.2f).put("double", 12.2).put("partitioning", partitionVal).put("string", "def").put("id", id).put("seqno", seqno);
            srcRecord = new SourceRecord(partition, ccOffset, topic, id, Schema.STRING_SCHEMA, "key", VALUE_SCHEMA_2, data);
        }
        System.out.println("{\"task\": " + id + ", \"seqno\": " + seqno + "}");
        seqno++;
        count++;
        return Collections.singletonList(srcRecord);
    } else {
        throttler.throttle();
        return Collections.emptyList();
    }
}

Example 77 with SourceRecord

use of org.apache.kafka.connect.source.SourceRecord in project kafka by apache.

the class MirrorSourceTask method convertRecord.

// visible for testing
SourceRecord convertRecord(ConsumerRecord<byte[], byte[]> record) {
    String targetTopic = formatRemoteTopic(record.topic());
    Headers headers = convertHeaders(record);
    return new SourceRecord(MirrorUtils.wrapPartition(new TopicPartition(record.topic(), record.partition()), sourceClusterAlias), MirrorUtils.wrapOffset(record.offset()), targetTopic, record.partition(), Schema.OPTIONAL_BYTES_SCHEMA, record.key(), Schema.BYTES_SCHEMA, record.value(), record.timestamp(), headers);
}

Example 78 with SourceRecord

use of org.apache.kafka.connect.source.SourceRecord in project kafka by apache.

the class MirrorCheckpointTaskTest method testCheckpoint.

@Test
public void testCheckpoint() {
    OffsetSyncStoreTest.FakeOffsetSyncStore offsetSyncStore = new OffsetSyncStoreTest.FakeOffsetSyncStore();
    MirrorCheckpointTask mirrorCheckpointTask = new MirrorCheckpointTask("source1", "target2", new DefaultReplicationPolicy(), offsetSyncStore, Collections.emptyMap(), Collections.emptyMap());
    offsetSyncStore.sync(new TopicPartition("topic1", 2), 3L, 4L);
    offsetSyncStore.sync(new TopicPartition("target2.topic5", 6), 7L, 8L);
    Checkpoint checkpoint1 = mirrorCheckpointTask.checkpoint("group9", new TopicPartition("topic1", 2), new OffsetAndMetadata(10, null));
    SourceRecord sourceRecord1 = mirrorCheckpointTask.checkpointRecord(checkpoint1, 123L);
    assertEquals(new TopicPartition("source1.topic1", 2), checkpoint1.topicPartition(), "checkpoint group9 source1.topic1 failed");
    assertEquals("group9", checkpoint1.consumerGroupId(), "checkpoint group9 consumerGroupId failed");
    assertEquals("group9", Checkpoint.unwrapGroup(sourceRecord1.sourcePartition()), "checkpoint group9 sourcePartition failed");
    assertEquals(10, checkpoint1.upstreamOffset(), "checkpoint group9 upstreamOffset failed");
    assertEquals(11, checkpoint1.downstreamOffset(), "checkpoint group9 downstreamOffset failed");
    assertEquals(123L, sourceRecord1.timestamp().longValue(), "checkpoint group9 timestamp failed");
    Checkpoint checkpoint2 = mirrorCheckpointTask.checkpoint("group11", new TopicPartition("target2.topic5", 6), new OffsetAndMetadata(12, null));
    SourceRecord sourceRecord2 = mirrorCheckpointTask.checkpointRecord(checkpoint2, 234L);
    assertEquals(new TopicPartition("topic5", 6), checkpoint2.topicPartition(), "checkpoint group11 topic5 failed");
    assertEquals("group11", checkpoint2.consumerGroupId(), "checkpoint group11 consumerGroupId failed");
    assertEquals("group11", Checkpoint.unwrapGroup(sourceRecord2.sourcePartition()), "checkpoint group11 sourcePartition failed");
    assertEquals(12, checkpoint2.upstreamOffset(), "checkpoint group11 upstreamOffset failed");
    assertEquals(13, checkpoint2.downstreamOffset(), "checkpoint group11 downstreamOffset failed");
    assertEquals(234L, sourceRecord2.timestamp().longValue(), "checkpoint group11 timestamp failed");
}

Example 79 with SourceRecord

use of org.apache.kafka.connect.source.SourceRecord in project kafka by apache.

the class MirrorSourceTaskTest method testPoll.

@Test
public void testPoll() {
    // Create a consumer mock
    byte[] key1 = "abc".getBytes();
    byte[] value1 = "fgh".getBytes();
    byte[] key2 = "123".getBytes();
    byte[] value2 = "456".getBytes();
    List<ConsumerRecord<byte[], byte[]>> consumerRecordsList = new ArrayList<>();
    String topicName = "test";
    String headerKey = "key";
    RecordHeaders headers = new RecordHeaders(new Header[] { new RecordHeader(headerKey, "value".getBytes()) });
    consumerRecordsList.add(new ConsumerRecord<>(topicName, 0, 0, System.currentTimeMillis(), TimestampType.CREATE_TIME, key1.length, value1.length, key1, value1, headers, Optional.empty()));
    consumerRecordsList.add(new ConsumerRecord<>(topicName, 1, 1, System.currentTimeMillis(), TimestampType.CREATE_TIME, key2.length, value2.length, key2, value2, headers, Optional.empty()));
    ConsumerRecords<byte[], byte[]> consumerRecords = new ConsumerRecords<>(Collections.singletonMap(new TopicPartition(topicName, 0), consumerRecordsList));
    @SuppressWarnings("unchecked") KafkaConsumer<byte[], byte[]> consumer = mock(KafkaConsumer.class);
    when(consumer.poll(any())).thenReturn(consumerRecords);
    MirrorMetrics metrics = mock(MirrorMetrics.class);
    String sourceClusterName = "cluster1";
    ReplicationPolicy replicationPolicy = new DefaultReplicationPolicy();
    MirrorSourceTask mirrorSourceTask = new MirrorSourceTask(consumer, metrics, sourceClusterName, replicationPolicy, 50);
    List<SourceRecord> sourceRecords = mirrorSourceTask.poll();
    assertEquals(2, sourceRecords.size());
    for (int i = 0; i < sourceRecords.size(); i++) {
        SourceRecord sourceRecord = sourceRecords.get(i);
        ConsumerRecord<byte[], byte[]> consumerRecord = consumerRecordsList.get(i);
        assertEquals(consumerRecord.key(), sourceRecord.key(), "consumerRecord key does not equal sourceRecord key");
        assertEquals(consumerRecord.value(), sourceRecord.value(), "consumerRecord value does not equal sourceRecord value");
        // We expect that the topicname will be based on the replication policy currently used
        assertEquals(replicationPolicy.formatRemoteTopic(sourceClusterName, topicName), sourceRecord.topic(), "topicName not the same as the current replicationPolicy");
        // We expect that MirrorMaker will keep the same partition assignment
        assertEquals(consumerRecord.partition(), sourceRecord.kafkaPartition().intValue(), "partition assignment not the same as the current replicationPolicy");
        // Check header values
        List<Header> expectedHeaders = new ArrayList<>();
        consumerRecord.headers().forEach(expectedHeaders::add);
        List<org.apache.kafka.connect.header.Header> taskHeaders = new ArrayList<>();
        sourceRecord.headers().forEach(taskHeaders::add);
        compareHeaders(expectedHeaders, taskHeaders);
    }
}

Example 80 with SourceRecord

use of org.apache.kafka.connect.source.SourceRecord in project kafka by apache.

the class FileStreamSourceTaskTest method testNormalLifecycle.

@Test
public void testNormalLifecycle() throws InterruptedException, IOException {
    expectOffsetLookupReturnNone();
    replay();
    task.start(config);
    OutputStream os = Files.newOutputStream(tempFile.toPath());
    assertNull(task.poll());
    os.write("partial line".getBytes());
    os.flush();
    assertNull(task.poll());
    os.write(" finished\n".getBytes());
    os.flush();
    List<SourceRecord> records = task.poll();
    assertEquals(1, records.size());
    assertEquals(TOPIC, records.get(0).topic());
    assertEquals("partial line finished", records.get(0).value());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.FILENAME_FIELD, tempFile.getAbsolutePath()), records.get(0).sourcePartition());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.POSITION_FIELD, 22L), records.get(0).sourceOffset());
    assertNull(task.poll());
    // Different line endings, and make sure the final \r doesn't result in a line until we can
    // read the subsequent byte.
    os.write("line1\rline2\r\nline3\nline4\n\r".getBytes());
    os.flush();
    records = task.poll();
    assertEquals(4, records.size());
    assertEquals("line1", records.get(0).value());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.FILENAME_FIELD, tempFile.getAbsolutePath()), records.get(0).sourcePartition());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.POSITION_FIELD, 28L), records.get(0).sourceOffset());
    assertEquals("line2", records.get(1).value());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.FILENAME_FIELD, tempFile.getAbsolutePath()), records.get(1).sourcePartition());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.POSITION_FIELD, 35L), records.get(1).sourceOffset());
    assertEquals("line3", records.get(2).value());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.FILENAME_FIELD, tempFile.getAbsolutePath()), records.get(2).sourcePartition());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.POSITION_FIELD, 41L), records.get(2).sourceOffset());
    assertEquals("line4", records.get(3).value());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.FILENAME_FIELD, tempFile.getAbsolutePath()), records.get(3).sourcePartition());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.POSITION_FIELD, 47L), records.get(3).sourceOffset());
    os.write("subsequent text".getBytes());
    os.flush();
    records = task.poll();
    assertEquals(1, records.size());
    assertEquals("", records.get(0).value());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.FILENAME_FIELD, tempFile.getAbsolutePath()), records.get(0).sourcePartition());
    assertEquals(Collections.singletonMap(FileStreamSourceTask.POSITION_FIELD, 48L), records.get(0).sourceOffset());
    os.close();
    task.stop();
}