Search in sources :

Example 6 with InternalTopicConfig

use of org.apache.kafka.streams.processor.internals.InternalTopicConfig in project kafka by apache.

the class TopologyBuilderTest method shouldAddInternalTopicConfigWithCleanupPolicyDeleteForInternalTopics.

@SuppressWarnings("unchecked")
@Test
public void shouldAddInternalTopicConfigWithCleanupPolicyDeleteForInternalTopics() throws Exception {
    final TopologyBuilder builder = new TopologyBuilder();
    builder.setApplicationId("appId");
    builder.addInternalTopic("foo");
    builder.addSource("source", "foo");
    final TopicsInfo topicsInfo = builder.topicGroups().values().iterator().next();
    final InternalTopicConfig topicConfig = topicsInfo.repartitionSourceTopics.get("appId-foo");
    final Properties properties = topicConfig.toProperties(0);
    assertEquals("appId-foo", topicConfig.name());
    assertEquals("delete", properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP));
    assertEquals(1, properties.size());
}
Also used : InternalTopicConfig(org.apache.kafka.streams.processor.internals.InternalTopicConfig) TopicsInfo(org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo) Properties(java.util.Properties) Test(org.junit.Test)

Example 7 with InternalTopicConfig

use of org.apache.kafka.streams.processor.internals.InternalTopicConfig in project kafka by apache.

the class TopologyBuilderTest method testTopicGroups.

@Test
public void testTopicGroups() {
    final TopologyBuilder builder = new TopologyBuilder();
    builder.setApplicationId("X");
    builder.addInternalTopic("topic-1x");
    builder.addSource("source-1", "topic-1", "topic-1x");
    builder.addSource("source-2", "topic-2");
    builder.addSource("source-3", "topic-3");
    builder.addSource("source-4", "topic-4");
    builder.addSource("source-5", "topic-5");
    builder.addProcessor("processor-1", new MockProcessorSupplier(), "source-1");
    builder.addProcessor("processor-2", new MockProcessorSupplier(), "source-2", "processor-1");
    builder.copartitionSources(mkList("source-1", "source-2"));
    builder.addProcessor("processor-3", new MockProcessorSupplier(), "source-3", "source-4");
    Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
    Map<Integer, TopicsInfo> expectedTopicGroups = new HashMap<>();
    expectedTopicGroups.put(0, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-1", "X-topic-1x", "topic-2"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
    expectedTopicGroups.put(1, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-3", "topic-4"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
    expectedTopicGroups.put(2, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-5"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
    assertEquals(3, topicGroups.size());
    assertEquals(expectedTopicGroups, topicGroups);
    Collection<Set<String>> copartitionGroups = builder.copartitionGroups();
    assertEquals(mkSet(mkSet("topic-1", "X-topic-1x", "topic-2")), new HashSet<>(copartitionGroups));
}
Also used : HashSet(java.util.HashSet) Utils.mkSet(org.apache.kafka.common.utils.Utils.mkSet) Set(java.util.Set) MockProcessorSupplier(org.apache.kafka.test.MockProcessorSupplier) HashMap(java.util.HashMap) InternalTopicConfig(org.apache.kafka.streams.processor.internals.InternalTopicConfig) TopicsInfo(org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo) Test(org.junit.Test)

Example 8 with InternalTopicConfig

use of org.apache.kafka.streams.processor.internals.InternalTopicConfig in project kafka by apache.

the class TopologyBuilderTest method shouldAddInternalTopicConfigWithCompactForNonWindowStores.

@SuppressWarnings("unchecked")
@Test
public void shouldAddInternalTopicConfigWithCompactForNonWindowStores() throws Exception {
    final TopologyBuilder builder = new TopologyBuilder();
    builder.setApplicationId("appId");
    builder.addSource("source", "topic");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source");
    builder.addStateStore(new MockStateStoreSupplier("name", true), "processor");
    final Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
    final TopicsInfo topicsInfo = topicGroups.values().iterator().next();
    final InternalTopicConfig topicConfig = topicsInfo.stateChangelogTopics.get("appId-name-changelog");
    final Properties properties = topicConfig.toProperties(0);
    assertEquals("appId-name-changelog", topicConfig.name());
    assertEquals("compact", properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP));
    assertEquals(1, properties.size());
}
Also used : MockProcessorSupplier(org.apache.kafka.test.MockProcessorSupplier) MockStateStoreSupplier(org.apache.kafka.test.MockStateStoreSupplier) InternalTopicConfig(org.apache.kafka.streams.processor.internals.InternalTopicConfig) TopicsInfo(org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo) Properties(java.util.Properties) Test(org.junit.Test)

Aggregations

InternalTopicConfig (org.apache.kafka.streams.processor.internals.InternalTopicConfig)8 TopicsInfo (org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo)5 Test (org.junit.Test)5 HashMap (java.util.HashMap)4 MockProcessorSupplier (org.apache.kafka.test.MockProcessorSupplier)4 Properties (java.util.Properties)3 HashSet (java.util.HashSet)2 Map (java.util.Map)2 Set (java.util.Set)2 MockStateStoreSupplier (org.apache.kafka.test.MockStateStoreSupplier)2 ArrayList (java.util.ArrayList)1 LinkedHashMap (java.util.LinkedHashMap)1 PartitionInfo (org.apache.kafka.common.PartitionInfo)1 Utils.mkSet (org.apache.kafka.common.utils.Utils.mkSet)1 RocksDBWindowStoreSupplier (org.apache.kafka.streams.state.internals.RocksDBWindowStoreSupplier)1 WindowStoreSupplier (org.apache.kafka.streams.state.internals.WindowStoreSupplier)1