Examples with BytesDeserializer org.apache.kafka.common.serialization.BytesDeserializer used on opensource projects

Example 1 with BytesDeserializer

use of org.apache.kafka.common.serialization.BytesDeserializer in project flink by apache.

the class KafkaContainerClient method readMessages.

public <T> List<T> readMessages(int expectedNumMessages, String groupId, String topic, Deserializer<T> valueDeserializer) throws Exception {
    Properties props = new Properties();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, container.getBootstrapServers());
    props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
    props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    final List<T> messages = Collections.synchronizedList(new ArrayList<>(expectedNumMessages));
    try (Consumer<Bytes, T> consumer = new KafkaConsumer<>(props, new BytesDeserializer(), valueDeserializer)) {
        waitUntilTopicAvailableThenAssign(topic, consumer, Duration.ofSeconds(60));
        // Keep polling until getting expected number of messages
        final Deadline deadline = Deadline.fromNow(Duration.ofSeconds(120));
        while (deadline.hasTimeLeft() && messages.size() < expectedNumMessages) {
            LOG.info("Waiting for messages. Received {}/{}.", messages.size(), expectedNumMessages);
            ConsumerRecords<Bytes, T> records = consumer.poll(Duration.ofMillis(1000));
            for (ConsumerRecord<Bytes, T> record : records) {
                messages.add(record.value());
            }
        }
        if (messages.size() != expectedNumMessages) {
            throw new IOException("Could not read expected number of messages.");
        }
        return messages;
    }
}

Example 2 with BytesDeserializer

use of org.apache.kafka.common.serialization.BytesDeserializer in project kafka by apache.

the class FetcherTest method testPreferredReadReplicaOffsetError.

@Test
public void testPreferredReadReplicaOffsetError() {
    buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis());
    subscriptions.assignFromUser(singleton(tp0));
    client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds));
    subscriptions.seek(tp0, 0);
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1)));
    consumerClient.poll(time.timer(0));
    assertTrue(fetcher.hasCompletedFetches());
    fetchedRecords();
    Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
    assertEquals(selected.id(), 1);
    // Return an error, should unset the preferred read replica
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.empty()));
    consumerClient.poll(time.timer(0));
    assertTrue(fetcher.hasCompletedFetches());
    fetchedRecords();
    selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
    assertEquals(selected.id(), -1);
}

Example 3 with BytesDeserializer

use of org.apache.kafka.common.serialization.BytesDeserializer in project kafka by apache.

the class FetcherTest method testPreferredReadReplica.

@Test
public void testPreferredReadReplica() {
    buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis());
    subscriptions.assignFromUser(singleton(tp0));
    client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds));
    subscriptions.seek(tp0, 0);
    // Node preferred replica before first fetch response
    Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
    assertEquals(selected.id(), -1);
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    // Set preferred read replica to node=1
    client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1)));
    consumerClient.poll(time.timer(0));
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
    assertTrue(partitionRecords.containsKey(tp0));
    // verify
    selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
    assertEquals(selected.id(), 1);
    assertEquals(1, fetcher.sendFetches());
    assertFalse(fetcher.hasCompletedFetches());
    // Set preferred read replica to node=2, which isn't in our metadata, should revert to leader
    client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(2)));
    consumerClient.poll(time.timer(0));
    assertTrue(fetcher.hasCompletedFetches());
    fetchedRecords();
    selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds());
    assertEquals(selected.id(), -1);
}