Examples with TopologyBuilder org.apache.kafka.streams.processor.TopologyBuilder used on opensource projects

Example 36 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project incubator-rya by apache.

the class StatementPatternProcessorIT method multiplePatterns_singleStatement.

@Test
public void multiplePatterns_singleStatement() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Setup a topology.
    final String query = "SELECT * WHERE { " + "?person <urn:talksTo> ?otherPerson . " + "?person ?action <urn:Bob>" + "}";
    final TopologyFactory factory = new TopologyFactory();
    final TopologyBuilder builder = factory.build(query, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Create some statements where some generates SP results and others do not.
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:talksTo"), vf.createURI("urn:Bob")), "a"));
    // Show the correct binding set results from the job.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final QueryBindingSet bs = new QueryBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("action", vf.createURI("urn:talksTo"));
    bs.addBinding("otherPerson", vf.createURI("urn:Bob"));
    expected.add(new VisibilityBindingSet(bs, "a"));
    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}

Example 37 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project apache-kafka-on-k8s by banzaicloud.

the class StreamThreadStateStoreProviderTest method before.

@SuppressWarnings("deprecation")
@Before
public void before() throws IOException {
    final TopologyBuilder builder = new TopologyBuilder();
    builder.addSource("the-source", topicName);
    builder.addProcessor("the-processor", new MockProcessorSupplier(), "the-source");
    builder.addStateStore(Stores.create("kv-store").withStringKeys().withStringValues().inMemory().build(), "the-processor");
    builder.addStateStore(Stores.create("window-store").withStringKeys().withStringValues().persistent().windowed(10, 10, 2, false).build(), "the-processor");
    final Properties properties = new Properties();
    final String applicationId = "applicationId";
    properties.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
    properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    stateDir = TestUtils.tempDirectory();
    properties.put(StreamsConfig.STATE_DIR_CONFIG, stateDir.getPath());
    final StreamsConfig streamsConfig = new StreamsConfig(properties);
    final MockClientSupplier clientSupplier = new MockClientSupplier();
    configureRestoreConsumer(clientSupplier, "applicationId-kv-store-changelog");
    configureRestoreConsumer(clientSupplier, "applicationId-window-store-changelog");
    builder.setApplicationId(applicationId);
    final ProcessorTopology topology = builder.build(null);
    tasks = new HashMap<>();
    stateDirectory = new StateDirectory(streamsConfig, new MockTime());
    taskOne = createStreamsTask(applicationId, streamsConfig, clientSupplier, topology, new TaskId(0, 0));
    taskOne.initializeStateStores();
    tasks.put(new TaskId(0, 0), taskOne);
    taskTwo = createStreamsTask(applicationId, streamsConfig, clientSupplier, topology, new TaskId(0, 1));
    taskTwo.initializeStateStores();
    tasks.put(new TaskId(0, 1), taskTwo);
    threadMock = EasyMock.createNiceMock(StreamThread.class);
    provider = new StreamThreadStateStoreProvider(threadMock);
}

Example 38 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project apache-kafka-on-k8s by banzaicloud.

the class ProcessorTopologyTest method shouldDriveGlobalStore.

@SuppressWarnings("unchecked")
@Test
public void shouldDriveGlobalStore() {
    final String storeName = "my-store";
    final StateStoreSupplier storeSupplier = Stores.create(storeName).withStringKeys().withStringValues().inMemory().disableLogging().build();
    final String global = "global";
    final String topic = "topic";
    final TopologyBuilder topologyBuilder = this.builder.addGlobalStore(storeSupplier, global, STRING_DESERIALIZER, STRING_DESERIALIZER, topic, "processor", define(new StatefulProcessor(storeName)));
    driver = new ProcessorTopologyTestDriver(config, topologyBuilder.internalTopologyBuilder);
    final KeyValueStore<String, String> globalStore = (KeyValueStore<String, String>) topologyBuilder.globalStateStores().get(storeName);
    driver.process(topic, "key1", "value1", STRING_SERIALIZER, STRING_SERIALIZER);
    driver.process(topic, "key2", "value2", STRING_SERIALIZER, STRING_SERIALIZER);
    assertEquals("value1", globalStore.get("key1"));
    assertEquals("value2", globalStore.get("key2"));
}