Examples with LogEntry org.apache.kafka.common.record.LogEntry used on opensource projects

Example 1 with LogEntry

use of org.apache.kafka.common.record.LogEntry in project kafka by apache.

the class Fetcher method parseCompletedFetch.

/**
     * The callback for fetch completion
     */
private PartitionRecords<K, V> parseCompletedFetch(CompletedFetch completedFetch) {
    TopicPartition tp = completedFetch.partition;
    FetchResponse.PartitionData partition = completedFetch.partitionData;
    long fetchOffset = completedFetch.fetchedOffset;
    int bytes = 0;
    int recordsCount = 0;
    PartitionRecords<K, V> parsedRecords = null;
    Errors error = partition.error;
    try {
        if (!subscriptions.isFetchable(tp)) {
            // this can happen when a rebalance happened or a partition consumption paused
            // while fetch is still in-flight
            log.debug("Ignoring fetched records for partition {} since it is no longer fetchable", tp);
        } else if (error == Errors.NONE) {
            // we are interested in this fetch only if the beginning offset matches the
            // current consumed position
            Long position = subscriptions.position(tp);
            if (position == null || position != fetchOffset) {
                log.debug("Discarding stale fetch response for partition {} since its offset {} does not match " + "the expected offset {}", tp, fetchOffset, position);
                return null;
            }
            List<ConsumerRecord<K, V>> parsed = new ArrayList<>();
            boolean skippedRecords = false;
            for (LogEntry logEntry : partition.records.deepEntries()) {
                // Skip the messages earlier than current position.
                if (logEntry.offset() >= position) {
                    parsed.add(parseRecord(tp, logEntry));
                    bytes += logEntry.sizeInBytes();
                } else
                    skippedRecords = true;
            }
            recordsCount = parsed.size();
            log.trace("Adding fetched record for partition {} with offset {} to buffered record list", tp, position);
            parsedRecords = new PartitionRecords<>(fetchOffset, tp, parsed);
            if (parsed.isEmpty() && !skippedRecords && (partition.records.sizeInBytes() > 0)) {
                if (completedFetch.responseVersion < 3) {
                    // Implement the pre KIP-74 behavior of throwing a RecordTooLargeException.
                    Map<TopicPartition, Long> recordTooLargePartitions = Collections.singletonMap(tp, fetchOffset);
                    throw new RecordTooLargeException("There are some messages at [Partition=Offset]: " + recordTooLargePartitions + " whose size is larger than the fetch size " + this.fetchSize + " and hence cannot be returned. Please considering upgrading your broker to 0.10.1.0 or " + "newer to avoid this issue. Alternately, increase the fetch size on the client (using " + ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG + ")", recordTooLargePartitions);
                } else {
                    // This should not happen with brokers that support FetchRequest/Response V3 or higher (i.e. KIP-74)
                    throw new KafkaException("Failed to make progress reading messages at " + tp + "=" + fetchOffset + ". Received a non-empty fetch response from the server, but no " + "complete records were found.");
                }
            }
            if (partition.highWatermark >= 0) {
                log.trace("Received {} records in fetch response for partition {} with offset {}", parsed.size(), tp, position);
                subscriptions.updateHighWatermark(tp, partition.highWatermark);
            }
        } else if (error == Errors.NOT_LEADER_FOR_PARTITION) {
            log.debug("Error in fetch for partition {}: {}", tp, error.exceptionName());
            this.metadata.requestUpdate();
        } else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION) {
            log.warn("Received unknown topic or partition error in fetch for partition {}. The topic/partition " + "may not exist or the user may not have Describe access to it", tp);
            this.metadata.requestUpdate();
        } else if (error == Errors.OFFSET_OUT_OF_RANGE) {
            if (fetchOffset != subscriptions.position(tp)) {
                log.debug("Discarding stale fetch response for partition {} since the fetched offset {}" + "does not match the current offset {}", tp, fetchOffset, subscriptions.position(tp));
            } else if (subscriptions.hasDefaultOffsetResetPolicy()) {
                log.info("Fetch offset {} is out of range for partition {}, resetting offset", fetchOffset, tp);
                subscriptions.needOffsetReset(tp);
            } else {
                throw new OffsetOutOfRangeException(Collections.singletonMap(tp, fetchOffset));
            }
        } else if (error == Errors.TOPIC_AUTHORIZATION_FAILED) {
            log.warn("Not authorized to read from topic {}.", tp.topic());
            throw new TopicAuthorizationException(Collections.singleton(tp.topic()));
        } else if (error == Errors.UNKNOWN) {
            log.warn("Unknown error fetching data for topic-partition {}", tp);
        } else {
            throw new IllegalStateException("Unexpected error code " + error.code() + " while fetching data");
        }
    } finally {
        completedFetch.metricAggregator.record(tp, bytes, recordsCount);
    }
    // likely that partitions for the same topic can remain together (allowing for more efficient serialization).
    if (bytes > 0 || error != Errors.NONE)
        subscriptions.movePartitionToEnd(tp);
    return parsedRecords;
}

Example 2 with LogEntry

use of org.apache.kafka.common.record.LogEntry in project kafka by apache.

the class RecordAccumulatorTest method testLinger.

@Test
public void testLinger() throws Exception {
    long lingerMs = 10L;
    RecordAccumulator accum = new RecordAccumulator(1024, 10 * 1024, CompressionType.NONE, lingerMs, 100L, metrics, time);
    accum.append(tp1, 0L, key, value, null, maxBlockTimeMs);
    assertEquals("No partitions should be ready", 0, accum.ready(cluster, time.milliseconds()).readyNodes.size());
    time.sleep(10);
    assertEquals("Our partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
    List<ProducerBatch> batches = accum.drain(cluster, Collections.singleton(node1), Integer.MAX_VALUE, 0).get(node1.id());
    assertEquals(1, batches.size());
    ProducerBatch batch = batches.get(0);
    Iterator<LogEntry> iter = batch.records().deepEntries().iterator();
    LogEntry entry = iter.next();
    assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key());
    assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value());
    assertFalse("No more records", iter.hasNext());
}

Example 3 with LogEntry

use of org.apache.kafka.common.record.LogEntry in project kafka by apache.

the class RecordAccumulatorTest method testFull.

@Test
public void testFull() throws Exception {
    long now = time.milliseconds();
    int batchSize = 1024;
    RecordAccumulator accum = new RecordAccumulator(batchSize, 10L * batchSize, CompressionType.NONE, 10L, 100L, metrics, time);
    int appends = batchSize / msgSize;
    for (int i = 0; i < appends; i++) {
        // append to the first batch
        accum.append(tp1, 0L, key, value, null, maxBlockTimeMs);
        Deque<ProducerBatch> partitionBatches = accum.batches().get(tp1);
        assertEquals(1, partitionBatches.size());
        assertTrue(partitionBatches.peekFirst().isWritable());
        assertEquals("No partitions should be ready.", 0, accum.ready(cluster, now).readyNodes.size());
    }
    // this append doesn't fit in the first batch, so a new batch is created and the first batch is closed
    accum.append(tp1, 0L, key, value, null, maxBlockTimeMs);
    Deque<ProducerBatch> partitionBatches = accum.batches().get(tp1);
    assertEquals(2, partitionBatches.size());
    Iterator<ProducerBatch> partitionBatchesIterator = partitionBatches.iterator();
    assertFalse(partitionBatchesIterator.next().isWritable());
    assertTrue(partitionBatchesIterator.next().isWritable());
    assertEquals("Our partition's leader should be ready", Collections.singleton(node1), accum.ready(cluster, time.milliseconds()).readyNodes);
    List<ProducerBatch> batches = accum.drain(cluster, Collections.singleton(node1), Integer.MAX_VALUE, 0).get(node1.id());
    assertEquals(1, batches.size());
    ProducerBatch batch = batches.get(0);
    Iterator<LogEntry> iter = batch.records().deepEntries().iterator();
    for (int i = 0; i < appends; i++) {
        LogEntry entry = iter.next();
        assertEquals("Keys should match", ByteBuffer.wrap(key), entry.record().key());
        assertEquals("Values should match", ByteBuffer.wrap(value), entry.record().value());
    }
    assertFalse("No more records", iter.hasNext());
}

Example 4 with LogEntry

use of org.apache.kafka.common.record.LogEntry in project kafka by apache.

the class RecordAccumulatorTest method testStressfulSituation.

@SuppressWarnings("unused")
@Test
public void testStressfulSituation() throws Exception {
    final int numThreads = 5;
    final int msgs = 10000;
    final int numParts = 2;
    final RecordAccumulator accum = new RecordAccumulator(1024, 10 * 1024, CompressionType.NONE, 0L, 100L, metrics, time);
    List<Thread> threads = new ArrayList<>();
    for (int i = 0; i < numThreads; i++) {
        threads.add(new Thread() {

            public void run() {
                for (int i = 0; i < msgs; i++) {
                    try {
                        accum.append(new TopicPartition(topic, i % numParts), 0L, key, value, null, maxBlockTimeMs);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }
        });
    }
    for (Thread t : threads) t.start();
    int read = 0;
    long now = time.milliseconds();
    while (read < numThreads * msgs) {
        Set<Node> nodes = accum.ready(cluster, now).readyNodes;
        List<ProducerBatch> batches = accum.drain(cluster, nodes, 5 * 1024, 0).get(node1.id());
        if (batches != null) {
            for (ProducerBatch batch : batches) {
                for (LogEntry entry : batch.records().deepEntries()) read++;
                accum.deallocate(batch);
            }
        }
    }
    for (Thread t : threads) t.join();
}