use of org.apache.kafka.streams.processor.TopologyBuilder in project kafka by apache.
the class StreamPartitionAssignorTest method testOnAssignment.
@Test
public void testOnAssignment() throws Exception {
TopicPartition t2p3 = new TopicPartition("topic2", 3);
TopologyBuilder builder = new TopologyBuilder();
builder.addSource("source1", "topic1");
builder.addSource("source2", "topic2");
builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
UUID uuid = UUID.randomUUID();
String client1 = "client1";
StreamThread thread = new StreamThread(builder, config, mockClientSupplier, "test", client1, uuid, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
partitionAssignor.configure(config.getConsumerConfigs(thread, "test", client1));
List<TaskId> activeTaskList = Utils.mkList(task0, task3);
Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
Map<TaskId, Set<TopicPartition>> standbyTasks = new HashMap<>();
activeTasks.put(task0, Utils.mkSet(t1p0));
activeTasks.put(task3, Utils.mkSet(t2p3));
standbyTasks.put(task1, Utils.mkSet(t1p0));
standbyTasks.put(task2, Utils.mkSet(t2p0));
AssignmentInfo info = new AssignmentInfo(activeTaskList, standbyTasks, new HashMap<HostInfo, Set<TopicPartition>>());
PartitionAssignor.Assignment assignment = new PartitionAssignor.Assignment(Utils.mkList(t1p0, t2p3), info.encode());
partitionAssignor.onAssignment(assignment);
assertEquals(activeTasks, partitionAssignor.activeTasks());
assertEquals(standbyTasks, partitionAssignor.standbyTasks());
}
use of org.apache.kafka.streams.processor.TopologyBuilder in project kafka by apache.
the class StreamThreadTest method testMaybeCommit.
@Test
public void testMaybeCommit() throws Exception {
File baseDir = Files.createTempDirectory("test").toFile();
try {
final long commitInterval = 1000L;
Properties props = configProps();
props.setProperty(StreamsConfig.STATE_DIR_CONFIG, baseDir.getCanonicalPath());
props.setProperty(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, Long.toString(commitInterval));
StreamsConfig config = new StreamsConfig(props);
final MockTime mockTime = new MockTime();
TopologyBuilder builder = new TopologyBuilder().setApplicationId("X");
builder.addSource("source1", "topic1");
MockClientSupplier mockClientSupplier = new MockClientSupplier();
StreamThread thread = new StreamThread(builder, config, mockClientSupplier, applicationId, clientId, processId, new Metrics(), mockTime, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {
@Override
public void maybeCommit(long now) {
super.maybeCommit(now);
}
@Override
protected StreamTask createStreamTask(TaskId id, Collection<TopicPartition> partitionsForTask) {
ProcessorTopology topology = builder.build(id.topicGroupId);
return new TestStreamTask(id, applicationId, partitionsForTask, topology, consumer, producer, restoreConsumer, config, new MockStreamsMetrics(new Metrics()), stateDirectory);
}
};
initPartitionGrouper(config, thread, mockClientSupplier);
ConsumerRebalanceListener rebalanceListener = thread.rebalanceListener;
List<TopicPartition> revokedPartitions;
List<TopicPartition> assignedPartitions;
//
// Assign t1p1 and t1p2. This should create Task 1 & 2
//
revokedPartitions = Collections.emptyList();
assignedPartitions = Arrays.asList(t1p1, t1p2);
rebalanceListener.onPartitionsRevoked(revokedPartitions);
rebalanceListener.onPartitionsAssigned(assignedPartitions);
assertEquals(2, thread.tasks().size());
// no task is committed before the commit interval
mockTime.sleep(commitInterval - 10L);
thread.maybeCommit(mockTime.milliseconds());
for (StreamTask task : thread.tasks().values()) {
assertFalse(((TestStreamTask) task).committed);
}
// all tasks are committed after the commit interval
mockTime.sleep(11L);
thread.maybeCommit(mockTime.milliseconds());
for (StreamTask task : thread.tasks().values()) {
assertTrue(((TestStreamTask) task).committed);
((TestStreamTask) task).committed = false;
}
// no task is committed before the commit interval, again
mockTime.sleep(commitInterval - 10L);
thread.maybeCommit(mockTime.milliseconds());
for (StreamTask task : thread.tasks().values()) {
assertFalse(((TestStreamTask) task).committed);
}
// all tasks are committed after the commit interval, again
mockTime.sleep(11L);
thread.maybeCommit(mockTime.milliseconds());
for (StreamTask task : thread.tasks().values()) {
assertTrue(((TestStreamTask) task).committed);
((TestStreamTask) task).committed = false;
}
} finally {
Utils.delete(baseDir);
}
}
use of org.apache.kafka.streams.processor.TopologyBuilder in project kafka by apache.
the class StreamThreadTest method testMaybeClean.
@Test
public void testMaybeClean() throws Exception {
File baseDir = Files.createTempDirectory("test").toFile();
try {
final long cleanupDelay = 1000L;
Properties props = configProps();
props.setProperty(StreamsConfig.STATE_CLEANUP_DELAY_MS_CONFIG, Long.toString(cleanupDelay));
props.setProperty(StreamsConfig.STATE_DIR_CONFIG, baseDir.getCanonicalPath());
StreamsConfig config = new StreamsConfig(props);
File applicationDir = new File(baseDir, applicationId);
applicationDir.mkdir();
File stateDir1 = new File(applicationDir, task1.toString());
File stateDir2 = new File(applicationDir, task2.toString());
File stateDir3 = new File(applicationDir, task3.toString());
File extraDir = new File(applicationDir, "X");
stateDir1.mkdir();
stateDir2.mkdir();
stateDir3.mkdir();
extraDir.mkdir();
final MockTime mockTime = new MockTime();
TopologyBuilder builder = new TopologyBuilder().setApplicationId("X");
builder.addSource("source1", "topic1");
MockClientSupplier mockClientSupplier = new MockClientSupplier();
StreamThread thread = new StreamThread(builder, config, mockClientSupplier, applicationId, clientId, processId, new Metrics(), mockTime, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {
@Override
public void maybeClean(long now) {
super.maybeClean(now);
}
@Override
protected StreamTask createStreamTask(TaskId id, Collection<TopicPartition> partitionsForTask) {
ProcessorTopology topology = builder.build(id.topicGroupId);
return new TestStreamTask(id, applicationId, partitionsForTask, topology, consumer, producer, restoreConsumer, config, new MockStreamsMetrics(new Metrics()), stateDirectory);
}
};
initPartitionGrouper(config, thread, mockClientSupplier);
ConsumerRebalanceListener rebalanceListener = thread.rebalanceListener;
assertTrue(thread.tasks().isEmpty());
mockTime.sleep(cleanupDelay);
// all directories exist since an assignment didn't happen
assertTrue(stateDir1.exists());
assertTrue(stateDir2.exists());
assertTrue(stateDir3.exists());
assertTrue(extraDir.exists());
List<TopicPartition> revokedPartitions;
List<TopicPartition> assignedPartitions;
Map<TaskId, StreamTask> prevTasks;
//
// Assign t1p1 and t1p2. This should create task1 & task2
//
final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
activeTasks.put(task1, Collections.singleton(t1p1));
activeTasks.put(task2, Collections.singleton(t1p2));
thread.partitionAssignor(new MockStreamsPartitionAssignor(activeTasks));
revokedPartitions = Collections.emptyList();
assignedPartitions = Arrays.asList(t1p1, t1p2);
prevTasks = new HashMap<>(thread.tasks());
rebalanceListener.onPartitionsRevoked(revokedPartitions);
rebalanceListener.onPartitionsAssigned(assignedPartitions);
// there shouldn't be any previous task
assertTrue(prevTasks.isEmpty());
// task 1 & 2 are created
assertEquals(2, thread.tasks().size());
// all directories should still exit before the cleanup delay time
mockTime.sleep(cleanupDelay - 10L);
thread.maybeClean(mockTime.milliseconds());
assertTrue(stateDir1.exists());
assertTrue(stateDir2.exists());
assertTrue(stateDir3.exists());
assertTrue(extraDir.exists());
// all state directories except for task task2 & task3 will be removed. the extra directory should still exists
mockTime.sleep(11L);
thread.maybeClean(mockTime.milliseconds());
assertTrue(stateDir1.exists());
assertTrue(stateDir2.exists());
assertFalse(stateDir3.exists());
assertTrue(extraDir.exists());
//
// Revoke t1p1 and t1p2. This should remove task1 & task2
//
activeTasks.clear();
revokedPartitions = assignedPartitions;
assignedPartitions = Collections.emptyList();
prevTasks = new HashMap<>(thread.tasks());
rebalanceListener.onPartitionsRevoked(revokedPartitions);
rebalanceListener.onPartitionsAssigned(assignedPartitions);
// previous tasks should be committed
assertEquals(2, prevTasks.size());
for (StreamTask task : prevTasks.values()) {
assertTrue(((TestStreamTask) task).committed);
((TestStreamTask) task).committed = false;
}
// no task
assertTrue(thread.tasks().isEmpty());
// all state directories for task task1 & task2 still exist before the cleanup delay time
mockTime.sleep(cleanupDelay - 10L);
thread.maybeClean(mockTime.milliseconds());
assertTrue(stateDir1.exists());
assertTrue(stateDir2.exists());
assertFalse(stateDir3.exists());
assertTrue(extraDir.exists());
// all state directories for task task1 & task2 are removed
mockTime.sleep(11L);
thread.maybeClean(mockTime.milliseconds());
assertFalse(stateDir1.exists());
assertFalse(stateDir2.exists());
assertFalse(stateDir3.exists());
assertTrue(extraDir.exists());
} finally {
Utils.delete(baseDir);
}
}
use of org.apache.kafka.streams.processor.TopologyBuilder in project kafka by apache.
the class StreamThreadTest method testPartitionAssignmentChange.
@SuppressWarnings("unchecked")
@Test
public void testPartitionAssignmentChange() throws Exception {
StreamsConfig config = new StreamsConfig(configProps());
StateListenerStub stateListener = new StateListenerStub();
TopologyBuilder builder = new TopologyBuilder().setApplicationId("X");
builder.addSource("source1", "topic1");
builder.addSource("source2", "topic2");
builder.addSource("source3", "topic3");
builder.addProcessor("processor", new MockProcessorSupplier(), "source2", "source3");
MockClientSupplier mockClientSupplier = new MockClientSupplier();
StreamThread thread = new StreamThread(builder, config, mockClientSupplier, applicationId, clientId, processId, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {
@Override
protected StreamTask createStreamTask(TaskId id, Collection<TopicPartition> partitionsForTask) {
ProcessorTopology topology = builder.build(id.topicGroupId);
return new TestStreamTask(id, applicationId, partitionsForTask, topology, consumer, producer, restoreConsumer, config, new MockStreamsMetrics(new Metrics()), stateDirectory);
}
};
thread.setStateListener(stateListener);
assertEquals(thread.state(), StreamThread.State.RUNNING);
initPartitionGrouper(config, thread, mockClientSupplier);
ConsumerRebalanceListener rebalanceListener = thread.rebalanceListener;
assertTrue(thread.tasks().isEmpty());
List<TopicPartition> revokedPartitions;
List<TopicPartition> assignedPartitions;
Set<TopicPartition> expectedGroup1;
Set<TopicPartition> expectedGroup2;
revokedPartitions = Collections.emptyList();
assignedPartitions = Collections.singletonList(t1p1);
expectedGroup1 = new HashSet<>(Arrays.asList(t1p1));
rebalanceListener.onPartitionsRevoked(revokedPartitions);
assertEquals(thread.state(), StreamThread.State.PARTITIONS_REVOKED);
Assert.assertEquals(stateListener.numChanges, 1);
Assert.assertEquals(stateListener.oldState, StreamThread.State.RUNNING);
Assert.assertEquals(stateListener.newState, StreamThread.State.PARTITIONS_REVOKED);
rebalanceListener.onPartitionsAssigned(assignedPartitions);
assertEquals(thread.state(), StreamThread.State.RUNNING);
Assert.assertEquals(stateListener.numChanges, 3);
Assert.assertEquals(stateListener.oldState, StreamThread.State.ASSIGNING_PARTITIONS);
Assert.assertEquals(stateListener.newState, StreamThread.State.RUNNING);
assertTrue(thread.tasks().containsKey(task1));
assertEquals(expectedGroup1, thread.tasks().get(task1).partitions());
assertEquals(1, thread.tasks().size());
revokedPartitions = assignedPartitions;
assignedPartitions = Collections.singletonList(t1p2);
expectedGroup2 = new HashSet<>(Arrays.asList(t1p2));
rebalanceListener.onPartitionsRevoked(revokedPartitions);
assertFalse(thread.tasks().containsKey(task1));
assertEquals(0, thread.tasks().size());
rebalanceListener.onPartitionsAssigned(assignedPartitions);
assertTrue(thread.tasks().containsKey(task2));
assertEquals(expectedGroup2, thread.tasks().get(task2).partitions());
assertEquals(1, thread.tasks().size());
revokedPartitions = assignedPartitions;
assignedPartitions = Arrays.asList(t1p1, t1p2);
expectedGroup1 = new HashSet<>(Collections.singleton(t1p1));
expectedGroup2 = new HashSet<>(Collections.singleton(t1p2));
rebalanceListener.onPartitionsRevoked(revokedPartitions);
rebalanceListener.onPartitionsAssigned(assignedPartitions);
assertTrue(thread.tasks().containsKey(task1));
assertTrue(thread.tasks().containsKey(task2));
assertEquals(expectedGroup1, thread.tasks().get(task1).partitions());
assertEquals(expectedGroup2, thread.tasks().get(task2).partitions());
assertEquals(2, thread.tasks().size());
revokedPartitions = assignedPartitions;
assignedPartitions = Arrays.asList(t2p1, t2p2, t3p1, t3p2);
expectedGroup1 = new HashSet<>(Arrays.asList(t2p1, t3p1));
expectedGroup2 = new HashSet<>(Arrays.asList(t2p2, t3p2));
rebalanceListener.onPartitionsRevoked(revokedPartitions);
rebalanceListener.onPartitionsAssigned(assignedPartitions);
assertTrue(thread.tasks().containsKey(task4));
assertTrue(thread.tasks().containsKey(task5));
assertEquals(expectedGroup1, thread.tasks().get(task4).partitions());
assertEquals(expectedGroup2, thread.tasks().get(task5).partitions());
assertEquals(2, thread.tasks().size());
revokedPartitions = assignedPartitions;
assignedPartitions = Arrays.asList(t1p1, t2p1, t3p1);
expectedGroup1 = new HashSet<>(Arrays.asList(t1p1));
expectedGroup2 = new HashSet<>(Arrays.asList(t2p1, t3p1));
rebalanceListener.onPartitionsRevoked(revokedPartitions);
rebalanceListener.onPartitionsAssigned(assignedPartitions);
assertTrue(thread.tasks().containsKey(task1));
assertTrue(thread.tasks().containsKey(task4));
assertEquals(expectedGroup1, thread.tasks().get(task1).partitions());
assertEquals(expectedGroup2, thread.tasks().get(task4).partitions());
assertEquals(2, thread.tasks().size());
revokedPartitions = assignedPartitions;
assignedPartitions = Arrays.asList(t1p1, t2p1, t3p1);
expectedGroup1 = new HashSet<>(Arrays.asList(t1p1));
expectedGroup2 = new HashSet<>(Arrays.asList(t2p1, t3p1));
rebalanceListener.onPartitionsRevoked(revokedPartitions);
rebalanceListener.onPartitionsAssigned(assignedPartitions);
assertTrue(thread.tasks().containsKey(task1));
assertTrue(thread.tasks().containsKey(task4));
assertEquals(expectedGroup1, thread.tasks().get(task1).partitions());
assertEquals(expectedGroup2, thread.tasks().get(task4).partitions());
assertEquals(2, thread.tasks().size());
revokedPartitions = assignedPartitions;
assignedPartitions = Collections.emptyList();
rebalanceListener.onPartitionsRevoked(revokedPartitions);
rebalanceListener.onPartitionsAssigned(assignedPartitions);
assertTrue(thread.tasks().isEmpty());
thread.close();
assertTrue((thread.state() == StreamThread.State.PENDING_SHUTDOWN) || (thread.state() == StreamThread.State.NOT_RUNNING));
}
use of org.apache.kafka.streams.processor.TopologyBuilder in project apache-kafka-on-k8s by banzaicloud.
the class RegexSourceIntegrationTest method shouldAddStateStoreToRegexDefinedSource.
@SuppressWarnings("deprecation")
@Test
public void shouldAddStateStoreToRegexDefinedSource() throws InterruptedException {
final ProcessorSupplier<String, String> processorSupplier = new MockProcessorSupplier<>();
final MockStateStoreSupplier stateStoreSupplier = new MockStateStoreSupplier("testStateStore", false);
final long thirtySecondTimeout = 30 * 1000;
final TopologyBuilder builder = new TopologyBuilder().addSource("ingest", Pattern.compile("topic-\\d+")).addProcessor("my-processor", processorSupplier, "ingest").addStateStore(stateStoreSupplier, "my-processor");
streams = new KafkaStreams(builder, streamsConfiguration);
try {
streams.start();
final TestCondition stateStoreNameBoundToSourceTopic = new TestCondition() {
@Override
public boolean conditionMet() {
final Map<String, List<String>> stateStoreToSourceTopic = builder.stateStoreNameToSourceTopics();
final List<String> topicNamesList = stateStoreToSourceTopic.get("testStateStore");
return topicNamesList != null && !topicNamesList.isEmpty() && topicNamesList.get(0).equals("topic-1");
}
};
TestUtils.waitForCondition(stateStoreNameBoundToSourceTopic, thirtySecondTimeout, "Did not find topic: [topic-1] connected to state store: [testStateStore]");
} finally {
streams.close();
}
}
Aggregations