Examples with TopicPartition org.apache.kafka.common.TopicPartition used on opensource projects

Example 16 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class KafkaConsumerTest method testLeaveGroupTimeout.

@Test
public void testLeaveGroupTimeout() throws Exception {
    Map<TopicPartition, Errors> response = new HashMap<>();
    response.put(tp0, Errors.NONE);
    OffsetCommitResponse commitResponse = offsetCommitResponse(response);
    consumerCloseTest(5000, Arrays.asList(commitResponse), 5000, false);
}

Example 17 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class KafkaConsumerTest method fetchResponse.

private FetchResponse fetchResponse(Map<TopicPartition, FetchInfo> fetches) {
    LinkedHashMap<TopicPartition, PartitionData> tpResponses = new LinkedHashMap<>();
    for (Map.Entry<TopicPartition, FetchInfo> fetchEntry : fetches.entrySet()) {
        TopicPartition partition = fetchEntry.getKey();
        long fetchOffset = fetchEntry.getValue().offset;
        int fetchCount = fetchEntry.getValue().count;
        MemoryRecordsBuilder records = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, fetchOffset);
        for (int i = 0; i < fetchCount; i++) records.append(0L, ("key-" + i).getBytes(), ("value-" + i).getBytes());
        tpResponses.put(partition, new FetchResponse.PartitionData(Errors.NONE, 0, records.build()));
    }
    return new FetchResponse(tpResponses, 0);
}

Example 18 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class KafkaConsumerTest method testOffsetOfPausedPartitions.

@Test
public void testOffsetOfPausedPartitions() {
    int rebalanceTimeoutMs = 60000;
    int sessionTimeoutMs = 30000;
    int heartbeatIntervalMs = 3000;
    int autoCommitIntervalMs = 1000;
    Time time = new MockTime();
    Cluster cluster = TestUtils.singletonCluster(topic, 2);
    Node node = cluster.nodes().get(0);
    Metadata metadata = new Metadata(0, Long.MAX_VALUE);
    metadata.update(cluster, Collections.<String>emptySet(), time.milliseconds());
    MockClient client = new MockClient(time, metadata);
    client.setNode(node);
    PartitionAssignor assignor = new RangeAssignor();
    final KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, rebalanceTimeoutMs, sessionTimeoutMs, heartbeatIntervalMs, true, autoCommitIntervalMs);
    // lookup coordinator
    client.prepareResponseFrom(new GroupCoordinatorResponse(Errors.NONE, node), node);
    Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
    // manual assignment
    Set<TopicPartition> partitions = Utils.mkSet(tp0, tp1);
    consumer.assign(partitions);
    // verify consumer's assignment
    assertTrue(consumer.assignment().equals(partitions));
    consumer.pause(partitions);
    consumer.seekToEnd(partitions);
    // fetch and verify committed offset of two partitions
    Map<TopicPartition, Long> offsets = new HashMap<>();
    offsets.put(tp0, 0L);
    offsets.put(tp1, 0L);
    client.prepareResponseFrom(offsetResponse(offsets, Errors.NONE), coordinator);
    assertEquals(0, consumer.committed(tp0).offset());
    assertEquals(0, consumer.committed(tp1).offset());
    // fetch and verify consumer's position in the two partitions
    client.prepareResponse(listOffsetsResponse(Collections.singletonMap(tp0, 3L), Errors.NONE));
    client.prepareResponse(listOffsetsResponse(Collections.singletonMap(tp1, 3L), Errors.NONE));
    assertEquals(3L, consumer.position(tp0));
    assertEquals(3L, consumer.position(tp1));
    client.requests().clear();
    consumer.unsubscribe();
    consumer.close();
}

Example 19 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class KafkaConsumerTest method testSeekNegative.

@Test(expected = IllegalArgumentException.class)
public void testSeekNegative() {
    Properties props = new Properties();
    props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "testSeekNegative");
    props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9999");
    props.setProperty(ConsumerConfig.METRIC_REPORTER_CLASSES_CONFIG, MockMetricsReporter.class.getName());
    KafkaConsumer<byte[], byte[]> consumer = newConsumer();
    try {
        consumer.assign(Arrays.asList(new TopicPartition("nonExistTopic", 0)));
        consumer.seek(new TopicPartition("nonExistTopic", 0), -1);
    } finally {
        consumer.close();
    }
}

Example 20 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class MockConsumerTest method testSimpleMock.

@Test
public void testSimpleMock() {
    consumer.subscribe(Arrays.asList("test"), new NoOpConsumerRebalanceListener());
    assertEquals(0, consumer.poll(1000).count());
    consumer.rebalance(Arrays.asList(new TopicPartition("test", 0), new TopicPartition("test", 1)));
    // Mock consumers need to seek manually since they cannot automatically reset offsets
    HashMap<TopicPartition, Long> beginningOffsets = new HashMap<>();
    beginningOffsets.put(new TopicPartition("test", 0), 0L);
    beginningOffsets.put(new TopicPartition("test", 1), 0L);
    consumer.updateBeginningOffsets(beginningOffsets);
    consumer.seek(new TopicPartition("test", 0), 0);
    ConsumerRecord<String, String> rec1 = new ConsumerRecord<String, String>("test", 0, 0, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, "key1", "value1");
    ConsumerRecord<String, String> rec2 = new ConsumerRecord<String, String>("test", 0, 1, 0L, TimestampType.CREATE_TIME, 0L, 0, 0, "key2", "value2");
    consumer.addRecord(rec1);
    consumer.addRecord(rec2);
    ConsumerRecords<String, String> recs = consumer.poll(1);
    Iterator<ConsumerRecord<String, String>> iter = recs.iterator();
    assertEquals(rec1, iter.next());
    assertEquals(rec2, iter.next());
    assertFalse(iter.hasNext());
    assertEquals(2L, consumer.position(new TopicPartition("test", 0)));
    consumer.commitSync();
    assertEquals(2L, consumer.committed(new TopicPartition("test", 0)).offset());
}