Examples with Assignment org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment used on opensource projects

Example 6 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads.

@Test
public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() {
    builder.addSource(null, "source1", null, null, null, "topic1");
    builder.addSource(null, "source2", null, null, null, "topic2");
    builder.addProcessor("processor", new MockApiProcessorSupplier<>(), "source1");
    builder.addProcessor("processorII", new MockApiProcessorSupplier<>(), "source2");
    final List<PartitionInfo> localInfos = asList(new PartitionInfo("topic1", 0, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic1", 1, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic1", 2, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic1", 3, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic2", 0, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic2", 1, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic2", 2, Node.noNode(), new Node[0], new Node[0]), new PartitionInfo("topic2", 3, Node.noNode(), new Node[0], new Node[0]));
    final Cluster localMetadata = new Cluster("cluster", Collections.singletonList(Node.noNode()), localInfos, emptySet(), emptySet());
    final List<String> topics = asList("topic1", "topic2");
    configureDefault();
    subscriptions.put("consumer10", new Subscription(topics, defaultSubscriptionInfo.encode()));
    subscriptions.put("consumer11", new Subscription(topics, defaultSubscriptionInfo.encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(localMetadata, new GroupSubscription(subscriptions)).groupAssignment();
    // check assigned partitions
    assertEquals(mkSet(mkSet(t2p2, t1p0, t1p2, t2p0), mkSet(t1p1, t2p1, t1p3, t2p3)), mkSet(new HashSet<>(assignments.get("consumer10").partitions()), new HashSet<>(assignments.get("consumer11").partitions())));
    // the first consumer
    final AssignmentInfo info10 = AssignmentInfo.decode(assignments.get("consumer10").userData());
    final List<TaskId> expectedInfo10TaskIds = asList(TASK_0_0, TASK_0_2, TASK_1_0, TASK_1_2);
    assertEquals(expectedInfo10TaskIds, info10.activeTasks());
    // the second consumer
    final AssignmentInfo info11 = AssignmentInfo.decode(assignments.get("consumer11").userData());
    final List<TaskId> expectedInfo11TaskIds = asList(TASK_0_1, TASK_0_3, TASK_1_1, TASK_1_3);
    assertEquals(expectedInfo11TaskIds, info11.activeTasks());
}

Example 7 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class StreamsPartitionAssignorTest method testOnAssignment.

@Test
public void testOnAssignment() {
    taskManager = EasyMock.createStrictMock(TaskManager.class);
    final Map<HostInfo, Set<TopicPartition>> hostState = Collections.singletonMap(new HostInfo("localhost", 9090), mkSet(t3p0, t3p3));
    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    activeTasks.put(TASK_0_0, mkSet(t3p0));
    activeTasks.put(TASK_0_3, mkSet(t3p3));
    final Map<TaskId, Set<TopicPartition>> standbyTasks = new HashMap<>();
    standbyTasks.put(TASK_0_1, mkSet(t3p1));
    standbyTasks.put(TASK_0_2, mkSet(t3p2));
    taskManager.handleAssignment(activeTasks, standbyTasks);
    EasyMock.expectLastCall();
    streamsMetadataState = EasyMock.createStrictMock(StreamsMetadataState.class);
    final Capture<Cluster> capturedCluster = EasyMock.newCapture();
    streamsMetadataState.onChange(EasyMock.eq(hostState), EasyMock.anyObject(), EasyMock.capture(capturedCluster));
    EasyMock.expectLastCall();
    EasyMock.replay(streamsMetadataState);
    configureDefaultPartitionAssignor();
    final List<TaskId> activeTaskList = asList(TASK_0_0, TASK_0_3);
    final AssignmentInfo info = new AssignmentInfo(LATEST_SUPPORTED_VERSION, activeTaskList, standbyTasks, hostState, emptyMap(), 0);
    final Assignment assignment = new Assignment(asList(t3p0, t3p3), info.encode());
    partitionAssignor.onAssignment(assignment, null);
    EasyMock.verify(streamsMetadataState);
    EasyMock.verify(taskManager);
    assertEquals(singleton(t3p0.topic()), capturedCluster.getValue().topics());
    assertEquals(2, capturedCluster.getValue().partitionsForTopic(t3p0.topic()).size());
}

Example 8 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class StreamsPartitionAssignorTest method testAssignWithStates.

@Test
public void testAssignWithStates() {
    builder.addSource(null, "source1", null, null, null, "topic1");
    builder.addSource(null, "source2", null, null, null, "topic2");
    builder.addProcessor("processor-1", new MockApiProcessorSupplier<>(), "source1");
    builder.addStateStore(new MockKeyValueStoreBuilder("store1", false), "processor-1");
    builder.addProcessor("processor-2", new MockApiProcessorSupplier<>(), "source2");
    builder.addStateStore(new MockKeyValueStoreBuilder("store2", false), "processor-2");
    builder.addStateStore(new MockKeyValueStoreBuilder("store3", false), "processor-2");
    final List<String> topics = asList("topic1", "topic2");
    final List<TaskId> tasks = asList(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2);
    adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap(asList(APPLICATION_ID + "-store1-changelog", APPLICATION_ID + "-store2-changelog", APPLICATION_ID + "-store3-changelog"), asList(3, 3, 3)));
    configureDefault();
    subscriptions.put("consumer10", new Subscription(topics, defaultSubscriptionInfo.encode()));
    subscriptions.put("consumer11", new Subscription(topics, defaultSubscriptionInfo.encode()));
    subscriptions.put("consumer20", new Subscription(topics, getInfo(UUID_2, EMPTY_TASKS, EMPTY_TASKS).encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    // check assigned partition size: since there is no previous task and there are two sub-topologies the assignment is random so we cannot check exact match
    assertEquals(2, assignments.get("consumer10").partitions().size());
    assertEquals(2, assignments.get("consumer11").partitions().size());
    assertEquals(2, assignments.get("consumer20").partitions().size());
    final AssignmentInfo info10 = AssignmentInfo.decode(assignments.get("consumer10").userData());
    final AssignmentInfo info11 = AssignmentInfo.decode(assignments.get("consumer11").userData());
    final AssignmentInfo info20 = AssignmentInfo.decode(assignments.get("consumer20").userData());
    assertEquals(2, info10.activeTasks().size());
    assertEquals(2, info11.activeTasks().size());
    assertEquals(2, info20.activeTasks().size());
    final Set<TaskId> allTasks = new HashSet<>();
    allTasks.addAll(info10.activeTasks());
    allTasks.addAll(info11.activeTasks());
    allTasks.addAll(info20.activeTasks());
    assertEquals(new HashSet<>(tasks), allTasks);
    // check tasks for state topics
    final Map<Subtopology, InternalTopologyBuilder.TopicsInfo> topicGroups = builder.subtopologyToTopicsInfo();
    assertEquals(mkSet(TASK_0_0, TASK_0_1, TASK_0_2), tasksForState("store1", tasks, topicGroups));
    assertEquals(mkSet(TASK_1_0, TASK_1_1, TASK_1_2), tasksForState("store2", tasks, topicGroups));
    assertEquals(mkSet(TASK_1_0, TASK_1_1, TASK_1_2), tasksForState("store3", tasks, topicGroups));
}

Example 9 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldReturnInterleavedAssignmentWithUnrevokedPartitionsRemovedWhenNewConsumerJoins.

@Test
public void shouldReturnInterleavedAssignmentWithUnrevokedPartitionsRemovedWhenNewConsumerJoins() {
    builder.addSource(null, "source1", null, null, null, "topic1");
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2);
    subscriptions.put(CONSUMER_1, new Subscription(Collections.singletonList("topic1"), getInfo(UUID_1, allTasks, EMPTY_TASKS).encode(), asList(t1p0, t1p1, t1p2)));
    subscriptions.put(CONSUMER_2, new Subscription(Collections.singletonList("topic1"), getInfo(UUID_2, EMPTY_TASKS, EMPTY_TASKS).encode(), emptyList()));
    createMockTaskManager(allTasks, allTasks);
    configureDefaultPartitionAssignor();
    final Map<String, Assignment> assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    assertThat(assignment.size(), equalTo(2));
    // The new consumer's assignment should be empty until c1 has the chance to revoke its partitions/tasks
    assertThat(assignment.get(CONSUMER_2).partitions(), equalTo(emptyList()));
    final AssignmentInfo actualAssignment = AssignmentInfo.decode(assignment.get(CONSUMER_2).userData());
    assertThat(actualAssignment.version(), is(LATEST_SUPPORTED_VERSION));
    assertThat(actualAssignment.activeTasks(), empty());
    // Note we're not asserting anything about standbys. If the assignor gave an active task to CONSUMER_2, it would
    // be converted to a standby, but we don't know whether the assignor will do that.
    assertThat(actualAssignment.partitionsByHost(), anEmptyMap());
    assertThat(actualAssignment.standbyPartitionByHost(), anEmptyMap());
    assertThat(actualAssignment.errCode(), is(0));
}

Example 10 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldReturnInterleavedAssignmentForOnlyFutureInstancesDuringVersionProbing.

@Test
public void shouldReturnInterleavedAssignmentForOnlyFutureInstancesDuringVersionProbing() {
    builder.addSource(null, "source1", null, null, null, "topic1");
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2);
    subscriptions.put(CONSUMER_1, new Subscription(Collections.singletonList("topic1"), encodeFutureSubscription(), emptyList()));
    subscriptions.put(CONSUMER_2, new Subscription(Collections.singletonList("topic1"), encodeFutureSubscription(), emptyList()));
    createMockTaskManager(allTasks, allTasks);
    configurePartitionAssignorWith(Collections.singletonMap(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1));
    final Map<String, Assignment> assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    assertThat(assignment.size(), equalTo(2));
    assertThat(assignment.get(CONSUMER_1).partitions(), equalTo(asList(t1p0, t1p2)));
    assertThat(AssignmentInfo.decode(assignment.get(CONSUMER_1).userData()), equalTo(new AssignmentInfo(LATEST_SUPPORTED_VERSION, asList(TASK_0_0, TASK_0_2), emptyMap(), emptyMap(), emptyMap(), 0)));
    assertThat(assignment.get(CONSUMER_2).partitions(), equalTo(Collections.singletonList(t1p1)));
    assertThat(AssignmentInfo.decode(assignment.get(CONSUMER_2).userData()), equalTo(new AssignmentInfo(LATEST_SUPPORTED_VERSION, Collections.singletonList(TASK_0_1), emptyMap(), emptyMap(), emptyMap(), 0)));
}