use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.
the class TopologyBuilderTest method shouldSetCorrectSourceNodesWithRegexUpdatedTopics.
@SuppressWarnings("unchecked")
@Test
public void shouldSetCorrectSourceNodesWithRegexUpdatedTopics() throws Exception {
final TopologyBuilder builder = new TopologyBuilder();
builder.addSource("source-1", "topic-foo");
builder.addSource("source-2", Pattern.compile("topic-[A-C]"));
builder.addSource("source-3", Pattern.compile("topic-\\d"));
StreamPartitionAssignor.SubscriptionUpdates subscriptionUpdates = new StreamPartitionAssignor.SubscriptionUpdates();
Field updatedTopicsField = subscriptionUpdates.getClass().getDeclaredField("updatedTopicSubscriptions");
updatedTopicsField.setAccessible(true);
Set<String> updatedTopics = (Set<String>) updatedTopicsField.get(subscriptionUpdates);
updatedTopics.add("topic-B");
updatedTopics.add("topic-3");
updatedTopics.add("topic-A");
builder.updateSubscriptions(subscriptionUpdates, null);
builder.setApplicationId("test-id");
Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
assertTrue(topicGroups.get(0).sourceTopics.contains("topic-foo"));
assertTrue(topicGroups.get(1).sourceTopics.contains("topic-A"));
assertTrue(topicGroups.get(1).sourceTopics.contains("topic-B"));
assertTrue(topicGroups.get(2).sourceTopics.contains("topic-3"));
}
use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.
the class TopologyBuilderTest method shouldAddInternalTopicConfigWithCompactAndDeleteSetForWindowStores.
@SuppressWarnings("unchecked")
@Test
public void shouldAddInternalTopicConfigWithCompactAndDeleteSetForWindowStores() throws Exception {
final TopologyBuilder builder = new TopologyBuilder();
builder.setApplicationId("appId");
builder.addSource("source", "topic");
builder.addProcessor("processor", new MockProcessorSupplier(), "source");
builder.addStateStore(new RocksDBWindowStoreSupplier("store", 30000, 3, false, null, null, 10000, true, Collections.<String, String>emptyMap(), false), "processor");
final Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
final TopicsInfo topicsInfo = topicGroups.values().iterator().next();
final InternalTopicConfig topicConfig = topicsInfo.stateChangelogTopics.get("appId-store-changelog");
final Properties properties = topicConfig.toProperties(0);
final List<String> policies = Arrays.asList(properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP).split(","));
assertEquals("appId-store-changelog", topicConfig.name());
assertTrue(policies.contains("compact"));
assertTrue(policies.contains("delete"));
assertEquals(2, policies.size());
assertEquals("30000", properties.getProperty(InternalTopicManager.RETENTION_MS));
assertEquals(2, properties.size());
}
use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.
the class TopologyBuilderTest method testTopicGroupsByStateStore.
@Test
public void testTopicGroupsByStateStore() {
final TopologyBuilder builder = new TopologyBuilder();
builder.setApplicationId("X");
builder.addSource("source-1", "topic-1", "topic-1x");
builder.addSource("source-2", "topic-2");
builder.addSource("source-3", "topic-3");
builder.addSource("source-4", "topic-4");
builder.addSource("source-5", "topic-5");
builder.addProcessor("processor-1", new MockProcessorSupplier(), "source-1");
builder.addProcessor("processor-2", new MockProcessorSupplier(), "source-2");
builder.addStateStore(new MockStateStoreSupplier("store-1", false), "processor-1", "processor-2");
builder.addProcessor("processor-3", new MockProcessorSupplier(), "source-3");
builder.addProcessor("processor-4", new MockProcessorSupplier(), "source-4");
builder.addStateStore(new MockStateStoreSupplier("store-2", false), "processor-3", "processor-4");
builder.addProcessor("processor-5", new MockProcessorSupplier(), "source-5");
StateStoreSupplier supplier = new MockStateStoreSupplier("store-3", false);
builder.addStateStore(supplier);
builder.connectProcessorAndStateStores("processor-5", "store-3");
Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
Map<Integer, TopicsInfo> expectedTopicGroups = new HashMap<>();
final String store1 = ProcessorStateManager.storeChangelogTopic("X", "store-1");
final String store2 = ProcessorStateManager.storeChangelogTopic("X", "store-2");
final String store3 = ProcessorStateManager.storeChangelogTopic("X", "store-3");
expectedTopicGroups.put(0, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-1", "topic-1x", "topic-2"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.singletonMap(store1, new InternalTopicConfig(store1, Collections.singleton(InternalTopicConfig.CleanupPolicy.compact), Collections.<String, String>emptyMap()))));
expectedTopicGroups.put(1, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-3", "topic-4"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.singletonMap(store2, new InternalTopicConfig(store2, Collections.singleton(InternalTopicConfig.CleanupPolicy.compact), Collections.<String, String>emptyMap()))));
expectedTopicGroups.put(2, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-5"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.singletonMap(store3, new InternalTopicConfig(store3, Collections.singleton(InternalTopicConfig.CleanupPolicy.compact), Collections.<String, String>emptyMap()))));
assertEquals(3, topicGroups.size());
assertEquals(expectedTopicGroups, topicGroups);
}
use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.
the class TopologyBuilderTest method shouldAddInternalTopicConfigWithCleanupPolicyDeleteForInternalTopics.
@SuppressWarnings("unchecked")
@Test
public void shouldAddInternalTopicConfigWithCleanupPolicyDeleteForInternalTopics() throws Exception {
final TopologyBuilder builder = new TopologyBuilder();
builder.setApplicationId("appId");
builder.addInternalTopic("foo");
builder.addSource("source", "foo");
final TopicsInfo topicsInfo = builder.topicGroups().values().iterator().next();
final InternalTopicConfig topicConfig = topicsInfo.repartitionSourceTopics.get("appId-foo");
final Properties properties = topicConfig.toProperties(0);
assertEquals("appId-foo", topicConfig.name());
assertEquals("delete", properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP));
assertEquals(1, properties.size());
}
use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.
the class TopologyBuilderTest method testTopicGroups.
@Test
public void testTopicGroups() {
final TopologyBuilder builder = new TopologyBuilder();
builder.setApplicationId("X");
builder.addInternalTopic("topic-1x");
builder.addSource("source-1", "topic-1", "topic-1x");
builder.addSource("source-2", "topic-2");
builder.addSource("source-3", "topic-3");
builder.addSource("source-4", "topic-4");
builder.addSource("source-5", "topic-5");
builder.addProcessor("processor-1", new MockProcessorSupplier(), "source-1");
builder.addProcessor("processor-2", new MockProcessorSupplier(), "source-2", "processor-1");
builder.copartitionSources(mkList("source-1", "source-2"));
builder.addProcessor("processor-3", new MockProcessorSupplier(), "source-3", "source-4");
Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
Map<Integer, TopicsInfo> expectedTopicGroups = new HashMap<>();
expectedTopicGroups.put(0, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-1", "X-topic-1x", "topic-2"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
expectedTopicGroups.put(1, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-3", "topic-4"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
expectedTopicGroups.put(2, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-5"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
assertEquals(3, topicGroups.size());
assertEquals(expectedTopicGroups, topicGroups);
Collection<Set<String>> copartitionGroups = builder.copartitionGroups();
assertEquals(mkSet(mkSet("topic-1", "X-topic-1x", "topic-2")), new HashSet<>(copartitionGroups));
}
Aggregations