Examples with TopicsInfo org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo used on opensource projects

Example 1 with TopicsInfo

use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.

the class TopologyBuilderTest method shouldSetCorrectSourceNodesWithRegexUpdatedTopics.

@SuppressWarnings("unchecked")
@Test
public void shouldSetCorrectSourceNodesWithRegexUpdatedTopics() throws Exception {
    final TopologyBuilder builder = new TopologyBuilder();
    builder.addSource("source-1", "topic-foo");
    builder.addSource("source-2", Pattern.compile("topic-[A-C]"));
    builder.addSource("source-3", Pattern.compile("topic-\\d"));
    StreamPartitionAssignor.SubscriptionUpdates subscriptionUpdates = new StreamPartitionAssignor.SubscriptionUpdates();
    Field updatedTopicsField = subscriptionUpdates.getClass().getDeclaredField("updatedTopicSubscriptions");
    updatedTopicsField.setAccessible(true);
    Set<String> updatedTopics = (Set<String>) updatedTopicsField.get(subscriptionUpdates);
    updatedTopics.add("topic-B");
    updatedTopics.add("topic-3");
    updatedTopics.add("topic-A");
    builder.updateSubscriptions(subscriptionUpdates, null);
    builder.setApplicationId("test-id");
    Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
    assertTrue(topicGroups.get(0).sourceTopics.contains("topic-foo"));
    assertTrue(topicGroups.get(1).sourceTopics.contains("topic-A"));
    assertTrue(topicGroups.get(1).sourceTopics.contains("topic-B"));
    assertTrue(topicGroups.get(2).sourceTopics.contains("topic-3"));
}

Example 2 with TopicsInfo

use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.

the class TopologyBuilderTest method shouldAddInternalTopicConfigWithCompactAndDeleteSetForWindowStores.

@SuppressWarnings("unchecked")
@Test
public void shouldAddInternalTopicConfigWithCompactAndDeleteSetForWindowStores() throws Exception {
    final TopologyBuilder builder = new TopologyBuilder();
    builder.setApplicationId("appId");
    builder.addSource("source", "topic");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source");
    builder.addStateStore(new RocksDBWindowStoreSupplier("store", 30000, 3, false, null, null, 10000, true, Collections.<String, String>emptyMap(), false), "processor");
    final Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
    final TopicsInfo topicsInfo = topicGroups.values().iterator().next();
    final InternalTopicConfig topicConfig = topicsInfo.stateChangelogTopics.get("appId-store-changelog");
    final Properties properties = topicConfig.toProperties(0);
    final List<String> policies = Arrays.asList(properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP).split(","));
    assertEquals("appId-store-changelog", topicConfig.name());
    assertTrue(policies.contains("compact"));
    assertTrue(policies.contains("delete"));
    assertEquals(2, policies.size());
    assertEquals("30000", properties.getProperty(InternalTopicManager.RETENTION_MS));
    assertEquals(2, properties.size());
}

Example 3 with TopicsInfo

use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.

the class TopologyBuilderTest method testTopicGroupsByStateStore.

@Test
public void testTopicGroupsByStateStore() {
    final TopologyBuilder builder = new TopologyBuilder();
    builder.setApplicationId("X");
    builder.addSource("source-1", "topic-1", "topic-1x");
    builder.addSource("source-2", "topic-2");
    builder.addSource("source-3", "topic-3");
    builder.addSource("source-4", "topic-4");
    builder.addSource("source-5", "topic-5");
    builder.addProcessor("processor-1", new MockProcessorSupplier(), "source-1");
    builder.addProcessor("processor-2", new MockProcessorSupplier(), "source-2");
    builder.addStateStore(new MockStateStoreSupplier("store-1", false), "processor-1", "processor-2");
    builder.addProcessor("processor-3", new MockProcessorSupplier(), "source-3");
    builder.addProcessor("processor-4", new MockProcessorSupplier(), "source-4");
    builder.addStateStore(new MockStateStoreSupplier("store-2", false), "processor-3", "processor-4");
    builder.addProcessor("processor-5", new MockProcessorSupplier(), "source-5");
    StateStoreSupplier supplier = new MockStateStoreSupplier("store-3", false);
    builder.addStateStore(supplier);
    builder.connectProcessorAndStateStores("processor-5", "store-3");
    Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
    Map<Integer, TopicsInfo> expectedTopicGroups = new HashMap<>();
    final String store1 = ProcessorStateManager.storeChangelogTopic("X", "store-1");
    final String store2 = ProcessorStateManager.storeChangelogTopic("X", "store-2");
    final String store3 = ProcessorStateManager.storeChangelogTopic("X", "store-3");
    expectedTopicGroups.put(0, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-1", "topic-1x", "topic-2"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.singletonMap(store1, new InternalTopicConfig(store1, Collections.singleton(InternalTopicConfig.CleanupPolicy.compact), Collections.<String, String>emptyMap()))));
    expectedTopicGroups.put(1, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-3", "topic-4"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.singletonMap(store2, new InternalTopicConfig(store2, Collections.singleton(InternalTopicConfig.CleanupPolicy.compact), Collections.<String, String>emptyMap()))));
    expectedTopicGroups.put(2, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-5"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.singletonMap(store3, new InternalTopicConfig(store3, Collections.singleton(InternalTopicConfig.CleanupPolicy.compact), Collections.<String, String>emptyMap()))));
    assertEquals(3, topicGroups.size());
    assertEquals(expectedTopicGroups, topicGroups);
}

Example 4 with TopicsInfo

use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.

the class TopologyBuilderTest method shouldAddInternalTopicConfigWithCleanupPolicyDeleteForInternalTopics.

@SuppressWarnings("unchecked")
@Test
public void shouldAddInternalTopicConfigWithCleanupPolicyDeleteForInternalTopics() throws Exception {
    final TopologyBuilder builder = new TopologyBuilder();
    builder.setApplicationId("appId");
    builder.addInternalTopic("foo");
    builder.addSource("source", "foo");
    final TopicsInfo topicsInfo = builder.topicGroups().values().iterator().next();
    final InternalTopicConfig topicConfig = topicsInfo.repartitionSourceTopics.get("appId-foo");
    final Properties properties = topicConfig.toProperties(0);
    assertEquals("appId-foo", topicConfig.name());
    assertEquals("delete", properties.getProperty(InternalTopicManager.CLEANUP_POLICY_PROP));
    assertEquals(1, properties.size());
}

Example 5 with TopicsInfo

use of org.apache.kafka.streams.processor.TopologyBuilder.TopicsInfo in project kafka by apache.

the class TopologyBuilderTest method testTopicGroups.

@Test
public void testTopicGroups() {
    final TopologyBuilder builder = new TopologyBuilder();
    builder.setApplicationId("X");
    builder.addInternalTopic("topic-1x");
    builder.addSource("source-1", "topic-1", "topic-1x");
    builder.addSource("source-2", "topic-2");
    builder.addSource("source-3", "topic-3");
    builder.addSource("source-4", "topic-4");
    builder.addSource("source-5", "topic-5");
    builder.addProcessor("processor-1", new MockProcessorSupplier(), "source-1");
    builder.addProcessor("processor-2", new MockProcessorSupplier(), "source-2", "processor-1");
    builder.copartitionSources(mkList("source-1", "source-2"));
    builder.addProcessor("processor-3", new MockProcessorSupplier(), "source-3", "source-4");
    Map<Integer, TopicsInfo> topicGroups = builder.topicGroups();
    Map<Integer, TopicsInfo> expectedTopicGroups = new HashMap<>();
    expectedTopicGroups.put(0, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-1", "X-topic-1x", "topic-2"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
    expectedTopicGroups.put(1, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-3", "topic-4"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
    expectedTopicGroups.put(2, new TopicsInfo(Collections.<String>emptySet(), mkSet("topic-5"), Collections.<String, InternalTopicConfig>emptyMap(), Collections.<String, InternalTopicConfig>emptyMap()));
    assertEquals(3, topicGroups.size());
    assertEquals(expectedTopicGroups, topicGroups);
    Collection<Set<String>> copartitionGroups = builder.copartitionGroups();
    assertEquals(mkSet(mkSet("topic-1", "X-topic-1x", "topic-2")), new HashSet<>(copartitionGroups));
}