Examples with TopologyBuilder org.apache.kafka.streams.processor.TopologyBuilder used on opensource projects

Example 1 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project kafka by apache.

the class StreamPartitionAssignorTest method testOnAssignment.

@Test
public void testOnAssignment() throws Exception {
    TopicPartition t2p3 = new TopicPartition("topic2", 3);
    TopologyBuilder builder = new TopologyBuilder();
    builder.addSource("source1", "topic1");
    builder.addSource("source2", "topic2");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
    UUID uuid = UUID.randomUUID();
    String client1 = "client1";
    StreamThread thread = new StreamThread(builder, config, mockClientSupplier, "test", client1, uuid, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    partitionAssignor.configure(config.getConsumerConfigs(thread, "test", client1));
    List<TaskId> activeTaskList = Utils.mkList(task0, task3);
    Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    Map<TaskId, Set<TopicPartition>> standbyTasks = new HashMap<>();
    activeTasks.put(task0, Utils.mkSet(t1p0));
    activeTasks.put(task3, Utils.mkSet(t2p3));
    standbyTasks.put(task1, Utils.mkSet(t1p0));
    standbyTasks.put(task2, Utils.mkSet(t2p0));
    AssignmentInfo info = new AssignmentInfo(activeTaskList, standbyTasks, new HashMap<HostInfo, Set<TopicPartition>>());
    PartitionAssignor.Assignment assignment = new PartitionAssignor.Assignment(Utils.mkList(t1p0, t2p3), info.encode());
    partitionAssignor.onAssignment(assignment);
    assertEquals(activeTasks, partitionAssignor.activeTasks());
    assertEquals(standbyTasks, partitionAssignor.standbyTasks());
}

Example 2 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project kafka by apache.

the class StreamThreadTest method testMaybeCommit.

@Test
public void testMaybeCommit() throws Exception {
    File baseDir = Files.createTempDirectory("test").toFile();
    try {
        final long commitInterval = 1000L;
        Properties props = configProps();
        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, baseDir.getCanonicalPath());
        props.setProperty(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, Long.toString(commitInterval));
        StreamsConfig config = new StreamsConfig(props);
        final MockTime mockTime = new MockTime();
        TopologyBuilder builder = new TopologyBuilder().setApplicationId("X");
        builder.addSource("source1", "topic1");
        MockClientSupplier mockClientSupplier = new MockClientSupplier();
        StreamThread thread = new StreamThread(builder, config, mockClientSupplier, applicationId, clientId, processId, new Metrics(), mockTime, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {

            @Override
            public void maybeCommit(long now) {
                super.maybeCommit(now);
            }

            @Override
            protected StreamTask createStreamTask(TaskId id, Collection<TopicPartition> partitionsForTask) {
                ProcessorTopology topology = builder.build(id.topicGroupId);
                return new TestStreamTask(id, applicationId, partitionsForTask, topology, consumer, producer, restoreConsumer, config, new MockStreamsMetrics(new Metrics()), stateDirectory);
            }
        };
        initPartitionGrouper(config, thread, mockClientSupplier);
        ConsumerRebalanceListener rebalanceListener = thread.rebalanceListener;
        List<TopicPartition> revokedPartitions;
        List<TopicPartition> assignedPartitions;
        //
        // Assign t1p1 and t1p2. This should create Task 1 & 2
        //
        revokedPartitions = Collections.emptyList();
        assignedPartitions = Arrays.asList(t1p1, t1p2);
        rebalanceListener.onPartitionsRevoked(revokedPartitions);
        rebalanceListener.onPartitionsAssigned(assignedPartitions);
        assertEquals(2, thread.tasks().size());
        // no task is committed before the commit interval
        mockTime.sleep(commitInterval - 10L);
        thread.maybeCommit(mockTime.milliseconds());
        for (StreamTask task : thread.tasks().values()) {
            assertFalse(((TestStreamTask) task).committed);
        }
        // all tasks are committed after the commit interval
        mockTime.sleep(11L);
        thread.maybeCommit(mockTime.milliseconds());
        for (StreamTask task : thread.tasks().values()) {
            assertTrue(((TestStreamTask) task).committed);
            ((TestStreamTask) task).committed = false;
        }
        // no task is committed before the commit interval, again
        mockTime.sleep(commitInterval - 10L);
        thread.maybeCommit(mockTime.milliseconds());
        for (StreamTask task : thread.tasks().values()) {
            assertFalse(((TestStreamTask) task).committed);
        }
        // all tasks are committed after the commit interval, again
        mockTime.sleep(11L);
        thread.maybeCommit(mockTime.milliseconds());
        for (StreamTask task : thread.tasks().values()) {
            assertTrue(((TestStreamTask) task).committed);
            ((TestStreamTask) task).committed = false;
        }
    } finally {
        Utils.delete(baseDir);
    }
}

Example 3 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project kafka by apache.

the class StreamThreadTest method testMaybeClean.

@Test
public void testMaybeClean() throws Exception {
    File baseDir = Files.createTempDirectory("test").toFile();
    try {
        final long cleanupDelay = 1000L;
        Properties props = configProps();
        props.setProperty(StreamsConfig.STATE_CLEANUP_DELAY_MS_CONFIG, Long.toString(cleanupDelay));
        props.setProperty(StreamsConfig.STATE_DIR_CONFIG, baseDir.getCanonicalPath());
        StreamsConfig config = new StreamsConfig(props);
        File applicationDir = new File(baseDir, applicationId);
        applicationDir.mkdir();
        File stateDir1 = new File(applicationDir, task1.toString());
        File stateDir2 = new File(applicationDir, task2.toString());
        File stateDir3 = new File(applicationDir, task3.toString());
        File extraDir = new File(applicationDir, "X");
        stateDir1.mkdir();
        stateDir2.mkdir();
        stateDir3.mkdir();
        extraDir.mkdir();
        final MockTime mockTime = new MockTime();
        TopologyBuilder builder = new TopologyBuilder().setApplicationId("X");
        builder.addSource("source1", "topic1");
        MockClientSupplier mockClientSupplier = new MockClientSupplier();
        StreamThread thread = new StreamThread(builder, config, mockClientSupplier, applicationId, clientId, processId, new Metrics(), mockTime, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {

            @Override
            public void maybeClean(long now) {
                super.maybeClean(now);
            }

            @Override
            protected StreamTask createStreamTask(TaskId id, Collection<TopicPartition> partitionsForTask) {
                ProcessorTopology topology = builder.build(id.topicGroupId);
                return new TestStreamTask(id, applicationId, partitionsForTask, topology, consumer, producer, restoreConsumer, config, new MockStreamsMetrics(new Metrics()), stateDirectory);
            }
        };
        initPartitionGrouper(config, thread, mockClientSupplier);
        ConsumerRebalanceListener rebalanceListener = thread.rebalanceListener;
        assertTrue(thread.tasks().isEmpty());
        mockTime.sleep(cleanupDelay);
        // all directories exist since an assignment didn't happen
        assertTrue(stateDir1.exists());
        assertTrue(stateDir2.exists());
        assertTrue(stateDir3.exists());
        assertTrue(extraDir.exists());
        List<TopicPartition> revokedPartitions;
        List<TopicPartition> assignedPartitions;
        Map<TaskId, StreamTask> prevTasks;
        //
        // Assign t1p1 and t1p2. This should create task1 & task2
        //
        final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
        activeTasks.put(task1, Collections.singleton(t1p1));
        activeTasks.put(task2, Collections.singleton(t1p2));
        thread.partitionAssignor(new MockStreamsPartitionAssignor(activeTasks));
        revokedPartitions = Collections.emptyList();
        assignedPartitions = Arrays.asList(t1p1, t1p2);
        prevTasks = new HashMap<>(thread.tasks());
        rebalanceListener.onPartitionsRevoked(revokedPartitions);
        rebalanceListener.onPartitionsAssigned(assignedPartitions);
        // there shouldn't be any previous task
        assertTrue(prevTasks.isEmpty());
        // task 1 & 2 are created
        assertEquals(2, thread.tasks().size());
        // all directories should still exit before the cleanup delay time
        mockTime.sleep(cleanupDelay - 10L);
        thread.maybeClean(mockTime.milliseconds());
        assertTrue(stateDir1.exists());
        assertTrue(stateDir2.exists());
        assertTrue(stateDir3.exists());
        assertTrue(extraDir.exists());
        // all state directories except for task task2 & task3 will be removed. the extra directory should still exists
        mockTime.sleep(11L);
        thread.maybeClean(mockTime.milliseconds());
        assertTrue(stateDir1.exists());
        assertTrue(stateDir2.exists());
        assertFalse(stateDir3.exists());
        assertTrue(extraDir.exists());
        //
        // Revoke t1p1 and t1p2. This should remove task1 & task2
        //
        activeTasks.clear();
        revokedPartitions = assignedPartitions;
        assignedPartitions = Collections.emptyList();
        prevTasks = new HashMap<>(thread.tasks());
        rebalanceListener.onPartitionsRevoked(revokedPartitions);
        rebalanceListener.onPartitionsAssigned(assignedPartitions);
        // previous tasks should be committed
        assertEquals(2, prevTasks.size());
        for (StreamTask task : prevTasks.values()) {
            assertTrue(((TestStreamTask) task).committed);
            ((TestStreamTask) task).committed = false;
        }
        // no task
        assertTrue(thread.tasks().isEmpty());
        // all state directories for task task1 & task2 still exist before the cleanup delay time
        mockTime.sleep(cleanupDelay - 10L);
        thread.maybeClean(mockTime.milliseconds());
        assertTrue(stateDir1.exists());
        assertTrue(stateDir2.exists());
        assertFalse(stateDir3.exists());
        assertTrue(extraDir.exists());
        // all state directories for task task1 & task2 are removed
        mockTime.sleep(11L);
        thread.maybeClean(mockTime.milliseconds());
        assertFalse(stateDir1.exists());
        assertFalse(stateDir2.exists());
        assertFalse(stateDir3.exists());
        assertTrue(extraDir.exists());
    } finally {
        Utils.delete(baseDir);
    }
}

Example 4 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project kafka by apache.

the class StreamThreadTest method testPartitionAssignmentChange.

@SuppressWarnings("unchecked")
@Test
public void testPartitionAssignmentChange() throws Exception {
    StreamsConfig config = new StreamsConfig(configProps());
    StateListenerStub stateListener = new StateListenerStub();
    TopologyBuilder builder = new TopologyBuilder().setApplicationId("X");
    builder.addSource("source1", "topic1");
    builder.addSource("source2", "topic2");
    builder.addSource("source3", "topic3");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source2", "source3");
    MockClientSupplier mockClientSupplier = new MockClientSupplier();
    StreamThread thread = new StreamThread(builder, config, mockClientSupplier, applicationId, clientId, processId, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {

        @Override
        protected StreamTask createStreamTask(TaskId id, Collection<TopicPartition> partitionsForTask) {
            ProcessorTopology topology = builder.build(id.topicGroupId);
            return new TestStreamTask(id, applicationId, partitionsForTask, topology, consumer, producer, restoreConsumer, config, new MockStreamsMetrics(new Metrics()), stateDirectory);
        }
    };
    thread.setStateListener(stateListener);
    assertEquals(thread.state(), StreamThread.State.RUNNING);
    initPartitionGrouper(config, thread, mockClientSupplier);
    ConsumerRebalanceListener rebalanceListener = thread.rebalanceListener;
    assertTrue(thread.tasks().isEmpty());
    List<TopicPartition> revokedPartitions;
    List<TopicPartition> assignedPartitions;
    Set<TopicPartition> expectedGroup1;
    Set<TopicPartition> expectedGroup2;
    revokedPartitions = Collections.emptyList();
    assignedPartitions = Collections.singletonList(t1p1);
    expectedGroup1 = new HashSet<>(Arrays.asList(t1p1));
    rebalanceListener.onPartitionsRevoked(revokedPartitions);
    assertEquals(thread.state(), StreamThread.State.PARTITIONS_REVOKED);
    Assert.assertEquals(stateListener.numChanges, 1);
    Assert.assertEquals(stateListener.oldState, StreamThread.State.RUNNING);
    Assert.assertEquals(stateListener.newState, StreamThread.State.PARTITIONS_REVOKED);
    rebalanceListener.onPartitionsAssigned(assignedPartitions);
    assertEquals(thread.state(), StreamThread.State.RUNNING);
    Assert.assertEquals(stateListener.numChanges, 3);
    Assert.assertEquals(stateListener.oldState, StreamThread.State.ASSIGNING_PARTITIONS);
    Assert.assertEquals(stateListener.newState, StreamThread.State.RUNNING);
    assertTrue(thread.tasks().containsKey(task1));
    assertEquals(expectedGroup1, thread.tasks().get(task1).partitions());
    assertEquals(1, thread.tasks().size());
    revokedPartitions = assignedPartitions;
    assignedPartitions = Collections.singletonList(t1p2);
    expectedGroup2 = new HashSet<>(Arrays.asList(t1p2));
    rebalanceListener.onPartitionsRevoked(revokedPartitions);
    assertFalse(thread.tasks().containsKey(task1));
    assertEquals(0, thread.tasks().size());
    rebalanceListener.onPartitionsAssigned(assignedPartitions);
    assertTrue(thread.tasks().containsKey(task2));
    assertEquals(expectedGroup2, thread.tasks().get(task2).partitions());
    assertEquals(1, thread.tasks().size());
    revokedPartitions = assignedPartitions;
    assignedPartitions = Arrays.asList(t1p1, t1p2);
    expectedGroup1 = new HashSet<>(Collections.singleton(t1p1));
    expectedGroup2 = new HashSet<>(Collections.singleton(t1p2));
    rebalanceListener.onPartitionsRevoked(revokedPartitions);
    rebalanceListener.onPartitionsAssigned(assignedPartitions);
    assertTrue(thread.tasks().containsKey(task1));
    assertTrue(thread.tasks().containsKey(task2));
    assertEquals(expectedGroup1, thread.tasks().get(task1).partitions());
    assertEquals(expectedGroup2, thread.tasks().get(task2).partitions());
    assertEquals(2, thread.tasks().size());
    revokedPartitions = assignedPartitions;
    assignedPartitions = Arrays.asList(t2p1, t2p2, t3p1, t3p2);
    expectedGroup1 = new HashSet<>(Arrays.asList(t2p1, t3p1));
    expectedGroup2 = new HashSet<>(Arrays.asList(t2p2, t3p2));
    rebalanceListener.onPartitionsRevoked(revokedPartitions);
    rebalanceListener.onPartitionsAssigned(assignedPartitions);
    assertTrue(thread.tasks().containsKey(task4));
    assertTrue(thread.tasks().containsKey(task5));
    assertEquals(expectedGroup1, thread.tasks().get(task4).partitions());
    assertEquals(expectedGroup2, thread.tasks().get(task5).partitions());
    assertEquals(2, thread.tasks().size());
    revokedPartitions = assignedPartitions;
    assignedPartitions = Arrays.asList(t1p1, t2p1, t3p1);
    expectedGroup1 = new HashSet<>(Arrays.asList(t1p1));
    expectedGroup2 = new HashSet<>(Arrays.asList(t2p1, t3p1));
    rebalanceListener.onPartitionsRevoked(revokedPartitions);
    rebalanceListener.onPartitionsAssigned(assignedPartitions);
    assertTrue(thread.tasks().containsKey(task1));
    assertTrue(thread.tasks().containsKey(task4));
    assertEquals(expectedGroup1, thread.tasks().get(task1).partitions());
    assertEquals(expectedGroup2, thread.tasks().get(task4).partitions());
    assertEquals(2, thread.tasks().size());
    revokedPartitions = assignedPartitions;
    assignedPartitions = Arrays.asList(t1p1, t2p1, t3p1);
    expectedGroup1 = new HashSet<>(Arrays.asList(t1p1));
    expectedGroup2 = new HashSet<>(Arrays.asList(t2p1, t3p1));
    rebalanceListener.onPartitionsRevoked(revokedPartitions);
    rebalanceListener.onPartitionsAssigned(assignedPartitions);
    assertTrue(thread.tasks().containsKey(task1));
    assertTrue(thread.tasks().containsKey(task4));
    assertEquals(expectedGroup1, thread.tasks().get(task1).partitions());
    assertEquals(expectedGroup2, thread.tasks().get(task4).partitions());
    assertEquals(2, thread.tasks().size());
    revokedPartitions = assignedPartitions;
    assignedPartitions = Collections.emptyList();
    rebalanceListener.onPartitionsRevoked(revokedPartitions);
    rebalanceListener.onPartitionsAssigned(assignedPartitions);
    assertTrue(thread.tasks().isEmpty());
    thread.close();
    assertTrue((thread.state() == StreamThread.State.PENDING_SHUTDOWN) || (thread.state() == StreamThread.State.NOT_RUNNING));
}

Example 5 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project apache-kafka-on-k8s by banzaicloud.

the class RegexSourceIntegrationTest method shouldAddStateStoreToRegexDefinedSource.

@SuppressWarnings("deprecation")
@Test
public void shouldAddStateStoreToRegexDefinedSource() throws InterruptedException {
    final ProcessorSupplier<String, String> processorSupplier = new MockProcessorSupplier<>();
    final MockStateStoreSupplier stateStoreSupplier = new MockStateStoreSupplier("testStateStore", false);
    final long thirtySecondTimeout = 30 * 1000;
    final TopologyBuilder builder = new TopologyBuilder().addSource("ingest", Pattern.compile("topic-\\d+")).addProcessor("my-processor", processorSupplier, "ingest").addStateStore(stateStoreSupplier, "my-processor");
    streams = new KafkaStreams(builder, streamsConfiguration);
    try {
        streams.start();
        final TestCondition stateStoreNameBoundToSourceTopic = new TestCondition() {

            @Override
            public boolean conditionMet() {
                final Map<String, List<String>> stateStoreToSourceTopic = builder.stateStoreNameToSourceTopics();
                final List<String> topicNamesList = stateStoreToSourceTopic.get("testStateStore");
                return topicNamesList != null && !topicNamesList.isEmpty() && topicNamesList.get(0).equals("topic-1");
            }
        };
        TestUtils.waitForCondition(stateStoreNameBoundToSourceTopic, thirtySecondTimeout, "Did not find topic: [topic-1] connected to state store: [testStateStore]");
    } finally {
        streams.close();
    }
}