Examples with SinkRecord org.apache.kafka.connect.sink.SinkRecord used on opensource projects

Example 16 with SinkRecord

use of org.apache.kafka.connect.sink.SinkRecord in project kafka by apache.

the class Worker method buildWorkerTask.

private WorkerTask buildWorkerTask(ConnectorConfig connConfig, ConnectorTaskId id, Task task, TaskStatus.Listener statusListener, TargetState initialState, Converter keyConverter, Converter valueConverter) {
    // Decide which type of worker task we need based on the type of task.
    if (task instanceof SourceTask) {
        TransformationChain<SourceRecord> transformationChain = new TransformationChain<>(connConfig.<SourceRecord>transformations());
        OffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetBackingStore, id.connector(), internalKeyConverter, internalValueConverter);
        OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetBackingStore, id.connector(), internalKeyConverter, internalValueConverter);
        KafkaProducer<byte[], byte[]> producer = new KafkaProducer<>(producerProps);
        return new WorkerSourceTask(id, (SourceTask) task, statusListener, initialState, keyConverter, valueConverter, transformationChain, producer, offsetReader, offsetWriter, config, time);
    } else if (task instanceof SinkTask) {
        TransformationChain<SinkRecord> transformationChain = new TransformationChain<>(connConfig.<SinkRecord>transformations());
        return new WorkerSinkTask(id, (SinkTask) task, statusListener, initialState, config, keyConverter, valueConverter, transformationChain, time);
    } else {
        log.error("Tasks must be a subclass of either SourceTask or SinkTask", task);
        throw new ConnectException("Tasks must be a subclass of either SourceTask or SinkTask");
    }
}

Example 17 with SinkRecord

use of org.apache.kafka.connect.sink.SinkRecord in project kafka by apache.

the class WorkerSinkTask method deliverMessages.

private void deliverMessages() {
    // Finally, deliver this batch to the sink
    try {
        // Since we reuse the messageBatch buffer, ensure we give the task its own copy
        task.put(new ArrayList<>(messageBatch));
        for (SinkRecord record : messageBatch) currentOffsets.put(new TopicPartition(record.topic(), record.kafkaPartition()), new OffsetAndMetadata(record.kafkaOffset() + 1));
        messageBatch.clear();
        // the task had not explicitly paused
        if (pausedForRedelivery) {
            if (!shouldPause())
                resumeAll();
            pausedForRedelivery = false;
        }
    } catch (RetriableException e) {
        log.error("RetriableException from SinkTask {}:", id, e);
        // If we're retrying a previous batch, make sure we've paused all topic partitions so we don't get new data,
        // but will still be able to poll in order to handle user-requested timeouts, keep group membership, etc.
        pausedForRedelivery = true;
        pauseAll();
    // Let this exit normally, the batch will be reprocessed on the next loop.
    } catch (Throwable t) {
        log.error("Task {} threw an uncaught and unrecoverable exception", id, t);
        log.error("Task is being killed and will not recover until manually restarted");
        throw new ConnectException("Exiting WorkerSinkTask due to unrecoverable exception.");
    }
}

Example 18 with SinkRecord

use of org.apache.kafka.connect.sink.SinkRecord in project kafka by apache.

the class WorkerSinkTaskTest method testIgnoredCommit.

@Test
public void testIgnoredCommit() throws Exception {
    expectInitializeTask();
    // iter 1
    expectPollInitialAssignment();
    // iter 2
    expectConsumerPoll(1);
    expectConversionAndTransformation(1);
    sinkTask.put(EasyMock.<Collection<SinkRecord>>anyObject());
    EasyMock.expectLastCall();
    final Map<TopicPartition, OffsetAndMetadata> workerStartingOffsets = new HashMap<>();
    workerStartingOffsets.put(TOPIC_PARTITION, new OffsetAndMetadata(FIRST_OFFSET));
    workerStartingOffsets.put(TOPIC_PARTITION2, new OffsetAndMetadata(FIRST_OFFSET));
    final Map<TopicPartition, OffsetAndMetadata> workerCurrentOffsets = new HashMap<>();
    workerCurrentOffsets.put(TOPIC_PARTITION, new OffsetAndMetadata(FIRST_OFFSET + 1));
    workerCurrentOffsets.put(TOPIC_PARTITION2, new OffsetAndMetadata(FIRST_OFFSET));
    // iter 3
    sinkTask.preCommit(workerCurrentOffsets);
    EasyMock.expectLastCall().andReturn(workerStartingOffsets);
    // no actual consumer.commit() triggered
    expectConsumerPoll(0);
    sinkTask.put(EasyMock.<Collection<SinkRecord>>anyObject());
    EasyMock.expectLastCall();
    PowerMock.replayAll();
    workerTask.initialize(TASK_CONFIG);
    workerTask.initializeAndStart();
    // iter 1 -- initial assignment
    workerTask.iteration();
    assertEquals(workerStartingOffsets, Whitebox.<Map<TopicPartition, OffsetAndMetadata>>getInternalState(workerTask, "currentOffsets"));
    assertEquals(workerStartingOffsets, Whitebox.<Map<TopicPartition, OffsetAndMetadata>>getInternalState(workerTask, "lastCommittedOffsets"));
    // iter 2 -- deliver 2 records
    workerTask.iteration();
    sinkTaskContext.getValue().requestCommit();
    // iter 3 -- commit
    workerTask.iteration();
    PowerMock.verifyAll();
}

Example 19 with SinkRecord

use of org.apache.kafka.connect.sink.SinkRecord in project kafka by apache.

the class WorkerSinkTaskTest method testMissingTimestampPropagation.

@Test
public void testMissingTimestampPropagation() throws Exception {
    expectInitializeTask();
    expectConsumerPoll(1, Record.NO_TIMESTAMP, TimestampType.CREATE_TIME);
    expectConversionAndTransformation(1);
    Capture<Collection<SinkRecord>> records = EasyMock.newCapture(CaptureType.ALL);
    sinkTask.put(EasyMock.capture(records));
    PowerMock.replayAll();
    workerTask.initialize(TASK_CONFIG);
    workerTask.initializeAndStart();
    workerTask.iteration();
    SinkRecord record = records.getValue().iterator().next();
    // we expect null for missing timestamp, the sentinel value of Record.NO_TIMESTAMP is Kafka's API
    assertEquals(null, record.timestamp());
    assertEquals(TimestampType.CREATE_TIME, record.timestampType());
    PowerMock.verifyAll();
}

Example 20 with SinkRecord

use of org.apache.kafka.connect.sink.SinkRecord in project kafka by apache.

the class ExtractFieldTest method withSchema.

@Test
public void withSchema() {
    final ExtractField<SinkRecord> xform = new ExtractField.Key<>();
    xform.configure(Collections.singletonMap("field", "magic"));
    final Schema keySchema = SchemaBuilder.struct().field("magic", Schema.INT32_SCHEMA).build();
    final Struct key = new Struct(keySchema).put("magic", 42);
    final SinkRecord record = new SinkRecord("test", 0, keySchema, key, null, null, 0);
    final SinkRecord transformedRecord = xform.apply(record);
    assertEquals(Schema.INT32_SCHEMA, transformedRecord.keySchema());
    assertEquals(42, transformedRecord.key());
}