Examples with KafkaRecordEntity org.apache.druid.data.input.kafka.KafkaRecordEntity used on opensource projects

Example 6 with KafkaRecordEntity

use of org.apache.druid.data.input.kafka.KafkaRecordEntity in project druid by druid-io.

the class KafkaInputFormatTest method testWithOutKey.

@Test
public // Headers cannot be null, so testing only no key use case!
void testWithOutKey() throws IOException {
    final byte[] payload = StringUtils.toUtf8("{\n" + "    \"timestamp\": \"2021-06-24\",\n" + "    \"bar\": null,\n" + "    \"foo\": \"x\",\n" + "    \"baz\": 4,\n" + "    \"o\": {\n" + "        \"mg\": 1\n" + "    }\n" + "}");
    Headers headers = new RecordHeaders(SAMPLE_HEADERS);
    inputEntity = new KafkaRecordEntity(new ConsumerRecord<byte[], byte[]>("sample", 0, 0, timestamp, null, null, 0, 0, null, payload, headers));
    final InputEntityReader reader = format.createReader(new InputRowSchema(new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo", "kafka.newheader.encoding", "kafka.newheader.kafkapkc", "kafka.newts.timestamp"))), ColumnsFilter.all()), inputEntity, null);
    final int numExpectedIterations = 1;
    try (CloseableIterator<InputRow> iterator = reader.read()) {
        int numActualIterations = 0;
        while (iterator.hasNext()) {
            final InputRow row = iterator.next();
            // Key verification
            Assert.assertTrue(row.getDimension("kafka.newkey.key").isEmpty());
            numActualIterations++;
        }
        Assert.assertEquals(numExpectedIterations, numActualIterations);
    }
}

Example 7 with KafkaRecordEntity

use of org.apache.druid.data.input.kafka.KafkaRecordEntity in project druid by druid-io.

the class KafkaStringHeaderFormatTest method testASCIIHeaderFormat.

@Test
public void testASCIIHeaderFormat() {
    Iterable<Header> header = ImmutableList.of(new Header() {

        @Override
        public String key() {
            return "encoding";
        }

        @Override
        public byte[] value() {
            return "application/json".getBytes(StandardCharsets.US_ASCII);
        }
    }, new Header() {

        @Override
        public String key() {
            return "kafkapkc";
        }

        @Override
        public byte[] value() {
            return "pkc-bar".getBytes(StandardCharsets.US_ASCII);
        }
    });
    String headerLabelPrefix = "test.kafka.header.";
    Headers headers = new RecordHeaders(header);
    inputEntity = new KafkaRecordEntity(new ConsumerRecord<byte[], byte[]>("sample", 0, 0, timestamp, null, null, 0, 0, null, "sampleValue".getBytes(StandardCharsets.UTF_8), headers));
    List<Pair<String, Object>> expectedResults = Arrays.asList(Pair.of("test.kafka.header.encoding", "application/json"), Pair.of("test.kafka.header.kafkapkc", "pkc-bar"));
    KafkaHeaderFormat headerInput = new KafkaStringHeaderFormat("US-ASCII");
    KafkaHeaderReader headerParser = headerInput.createReader(inputEntity.getRecord().headers(), headerLabelPrefix);
    List<Pair<String, Object>> rows = headerParser.read();
    Assert.assertEquals(expectedResults, rows);
}

Example 8 with KafkaRecordEntity

use of org.apache.druid.data.input.kafka.KafkaRecordEntity in project druid by druid-io.

the class KafkaStringHeaderFormatTest method testDefaultHeaderFormat.

@Test
public void testDefaultHeaderFormat() {
    String headerLabelPrefix = "test.kafka.header.";
    Headers headers = new RecordHeaders(SAMPLE_HEADERS);
    inputEntity = new KafkaRecordEntity(new ConsumerRecord<byte[], byte[]>("sample", 0, 0, timestamp, null, null, 0, 0, null, "sampleValue".getBytes(StandardCharsets.UTF_8), headers));
    List<Pair<String, Object>> expectedResults = Arrays.asList(Pair.of("test.kafka.header.encoding", "application/json"), Pair.of("test.kafka.header.kafkapkc", "pkc-bar"));
    KafkaHeaderFormat headerInput = new KafkaStringHeaderFormat(null);
    KafkaHeaderReader headerParser = headerInput.createReader(inputEntity.getRecord().headers(), headerLabelPrefix);
    Assert.assertEquals(expectedResults, headerParser.read());
}

Example 9 with KafkaRecordEntity

use of org.apache.druid.data.input.kafka.KafkaRecordEntity in project druid by druid-io.

the class IncrementalPublishingKafkaIndexTaskRunner method possiblyResetOffsetsOrWait.

private void possiblyResetOffsetsOrWait(Map<TopicPartition, Long> outOfRangePartitions, RecordSupplier<Integer, Long, KafkaRecordEntity> recordSupplier, TaskToolbox taskToolbox) throws InterruptedException, IOException {
    final Map<TopicPartition, Long> resetPartitions = new HashMap<>();
    boolean doReset = false;
    if (task.getTuningConfig().isResetOffsetAutomatically()) {
        for (Map.Entry<TopicPartition, Long> outOfRangePartition : outOfRangePartitions.entrySet()) {
            final TopicPartition topicPartition = outOfRangePartition.getKey();
            final long nextOffset = outOfRangePartition.getValue();
            // seek to the beginning to get the least available offset
            StreamPartition<Integer> streamPartition = StreamPartition.of(topicPartition.topic(), topicPartition.partition());
            final Long leastAvailableOffset = recordSupplier.getEarliestSequenceNumber(streamPartition);
            if (leastAvailableOffset == null) {
                throw new ISE("got null sequence number for partition[%s] when fetching from kafka!", topicPartition.partition());
            }
            // reset the seek
            recordSupplier.seek(streamPartition, nextOffset);
            // next message offset that we are trying to fetch
            if (leastAvailableOffset > nextOffset) {
                doReset = true;
                resetPartitions.put(topicPartition, nextOffset);
            }
        }
    }
    if (doReset) {
        sendResetRequestAndWait(CollectionUtils.mapKeys(resetPartitions, streamPartition -> StreamPartition.of(streamPartition.topic(), streamPartition.partition())), taskToolbox);
    } else {
        log.warn("Retrying in %dms", task.getPollRetryMs());
        pollRetryLock.lockInterruptibly();
        try {
            long nanos = TimeUnit.MILLISECONDS.toNanos(task.getPollRetryMs());
            while (nanos > 0L && !pauseRequested && !stopRequested.get()) {
                nanos = isAwaitingRetry.awaitNanos(nanos);
            }
        } finally {
            pollRetryLock.unlock();
        }
    }
}