Search in sources :

Example 26 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project incubator-rya by apache.

the class AggregationProcessorIT method sum.

@Test
public void sum() throws Exception {
    // A query that figures out how much food each person has.
    final String sparql = "SELECT ?person (sum(?foodCount) as ?totalFood) " + "WHERE { " + "?person <urn:hasFoodType> ?food . " + "?food <urn:count> ?foodCount . " + "} GROUP BY ?person";
    // Create the statements that will be input into the query..
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:hasFoodType"), vf.createURI("urn:corn")), ""));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:hasFoodType"), vf.createURI("urn:apple")), ""));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:corn"), vf.createURI("urn:count"), vf.createLiteral(4)), ""));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:apple"), vf.createURI("urn:count"), vf.createLiteral(3)), ""));
    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("totalFood", vf.createLiteral("4", XMLSchema.INTEGER));
    expected.add(new VisibilityBindingSet(bs, ""));
    bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("totalFood", vf.createLiteral("7", XMLSchema.INTEGER));
    expected.add(new VisibilityBindingSet(bs, ""));
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
Also used : VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) TopologyBuilder(org.apache.kafka.streams.processor.TopologyBuilder) ValueFactoryImpl(org.openrdf.model.impl.ValueFactoryImpl) ArrayList(java.util.ArrayList) ValueFactory(org.openrdf.model.ValueFactory) TopologyFactory(org.apache.rya.streams.kafka.topology.TopologyFactory) VisibilityStatement(org.apache.rya.api.model.VisibilityStatement) RandomUUIDFactory(org.apache.rya.api.function.projection.RandomUUIDFactory) MapBindingSet(org.openrdf.query.impl.MapBindingSet) UUID(java.util.UUID) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 27 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project incubator-rya by apache.

the class JoinProcessorIT method newRightResult.

@Test
public void newRightResult() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Setup a topology.
    final String query = "SELECT * WHERE { " + "?person <urn:talksTo> ?employee ." + "?employee <urn:worksAt> ?business" + " }";
    final TopologyFactory factory = new TopologyFactory();
    final TopologyBuilder builder = factory.build(query, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Create some statements that generate a bunch of right SP results.
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:worksAt"), vf.createURI("urn:TacoPlace")), "a&b"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Charlie"), vf.createURI("urn:worksAt"), vf.createURI("urn:BurgerJoint")), "a"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Eve"), vf.createURI("urn:worksAt"), vf.createURI("urn:CoffeeShop")), "b"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:worksAt"), vf.createURI("urn:BurgerJoint")), "b|c"));
    // Add a statement that will generate a left result that joins with some of those right results.
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:talksTo"), vf.createURI("urn:Bob")), "c"));
    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("employee", vf.createURI("urn:Bob"));
    bs.addBinding("business", vf.createURI("urn:TacoPlace"));
    expected.add(new VisibilityBindingSet(bs, "a&b&c"));
    bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("employee", vf.createURI("urn:Bob"));
    bs.addBinding("business", vf.createURI("urn:BurgerJoint"));
    expected.add(new VisibilityBindingSet(bs, "c&(b|c)"));
    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
Also used : VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) TopologyBuilder(org.apache.kafka.streams.processor.TopologyBuilder) ValueFactoryImpl(org.openrdf.model.impl.ValueFactoryImpl) ArrayList(java.util.ArrayList) TopologyFactory(org.apache.rya.streams.kafka.topology.TopologyFactory) ValueFactory(org.openrdf.model.ValueFactory) VisibilityStatement(org.apache.rya.api.model.VisibilityStatement) RandomUUIDFactory(org.apache.rya.api.function.projection.RandomUUIDFactory) MapBindingSet(org.openrdf.query.impl.MapBindingSet) UUID(java.util.UUID) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 28 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project incubator-rya by apache.

the class JoinProcessorIT method newLeftResult.

@Test
public void newLeftResult() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Setup a topology.
    final String query = "SELECT * WHERE { " + "?person <urn:talksTo> ?employee ." + "?employee <urn:worksAt> ?business" + " }";
    final TopologyFactory factory = new TopologyFactory();
    final TopologyBuilder builder = factory.build(query, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Create some statements that generate a bunch of right SP results.
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:worksAt"), vf.createURI("urn:TacoPlace")), "a&b"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Charlie"), vf.createURI("urn:worksAt"), vf.createURI("urn:BurgerJoint")), "a"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Eve"), vf.createURI("urn:worksAt"), vf.createURI("urn:CoffeeShop")), "b"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:worksAt"), vf.createURI("urn:BurgerJoint")), "b|c"));
    // Add a statement that will generate a left result that joins with some of those right results.
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:talksTo"), vf.createURI("urn:Bob")), "c"));
    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("employee", vf.createURI("urn:Bob"));
    bs.addBinding("business", vf.createURI("urn:TacoPlace"));
    expected.add(new VisibilityBindingSet(bs, "a&b&c"));
    bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("employee", vf.createURI("urn:Bob"));
    bs.addBinding("business", vf.createURI("urn:BurgerJoint"));
    expected.add(new VisibilityBindingSet(bs, "c&(b|c)"));
    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
Also used : VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) TopologyBuilder(org.apache.kafka.streams.processor.TopologyBuilder) ValueFactoryImpl(org.openrdf.model.impl.ValueFactoryImpl) ArrayList(java.util.ArrayList) TopologyFactory(org.apache.rya.streams.kafka.topology.TopologyFactory) ValueFactory(org.openrdf.model.ValueFactory) VisibilityStatement(org.apache.rya.api.model.VisibilityStatement) RandomUUIDFactory(org.apache.rya.api.function.projection.RandomUUIDFactory) MapBindingSet(org.openrdf.query.impl.MapBindingSet) UUID(java.util.UUID) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 29 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project incubator-rya by apache.

the class JoinProcessorIT method leftJoin.

@Test
public void leftJoin() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Setup a topology.
    final String query = "SELECT * WHERE { " + "?person <urn:talksTo> ?employee ." + "OPTIONAL{ ?employee <urn:worksAt> ?business } " + " }";
    final TopologyFactory factory = new TopologyFactory();
    final TopologyBuilder builder = factory.build(query, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Create some statements that generate a result that includes the optional value as well as one that does not.
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:talksTo"), vf.createURI("urn:Bob")), "a"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:worksAt"), vf.createURI("urn:TacoPlace")), "b"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:talksTo"), vf.createURI("urn:Charlie")), "c"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:David"), vf.createURI("urn:worksAt"), vf.createURI("urn:BurgerJoint")), "d"));
    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("employee", vf.createURI("urn:Bob"));
    expected.add(new VisibilityBindingSet(bs, "a"));
    bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("employee", vf.createURI("urn:Bob"));
    bs.addBinding("business", vf.createURI("urn:TacoPlace"));
    expected.add(new VisibilityBindingSet(bs, "a&b"));
    bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Bob"));
    bs.addBinding("employee", vf.createURI("urn:Charlie"));
    expected.add(new VisibilityBindingSet(bs, "c"));
    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
Also used : VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) TopologyBuilder(org.apache.kafka.streams.processor.TopologyBuilder) ValueFactoryImpl(org.openrdf.model.impl.ValueFactoryImpl) ArrayList(java.util.ArrayList) TopologyFactory(org.apache.rya.streams.kafka.topology.TopologyFactory) ValueFactory(org.openrdf.model.ValueFactory) VisibilityStatement(org.apache.rya.api.model.VisibilityStatement) RandomUUIDFactory(org.apache.rya.api.function.projection.RandomUUIDFactory) MapBindingSet(org.openrdf.query.impl.MapBindingSet) UUID(java.util.UUID) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 30 with TopologyBuilder

use of org.apache.kafka.streams.processor.TopologyBuilder in project incubator-rya by apache.

the class TemporalFilterIT method showWithinWorks.

@Test
public void showWithinWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Get the RDF model objects that will be used to build the query.
    final String sparql = "PREFIX time: <http://www.w3.org/2006/time/> \n" + "PREFIX tempf: <" + TemporalURIs.NAMESPACE + ">\n" + "SELECT * \n" + "WHERE { \n" + "  <urn:time> time:atTime ?date .\n" + " FILTER(tempf:within(?date, \"" + TIME.toString() + "/" + TIME_20.toString() + "\")) " + "}";
    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Create the statements that will be input into the query.
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = getStatements();
    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet bs = new MapBindingSet();
    bs.addBinding("date", vf.createLiteral(TIME_10.toString()));
    expected.add(new VisibilityBindingSet(bs, "a"));
    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
Also used : VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) TopologyBuilder(org.apache.kafka.streams.processor.TopologyBuilder) ValueFactoryImpl(org.openrdf.model.impl.ValueFactoryImpl) TopologyFactory(org.apache.rya.streams.kafka.topology.TopologyFactory) ValueFactory(org.openrdf.model.ValueFactory) VisibilityStatement(org.apache.rya.api.model.VisibilityStatement) RandomUUIDFactory(org.apache.rya.api.function.projection.RandomUUIDFactory) MapBindingSet(org.openrdf.query.impl.MapBindingSet) UUID(java.util.UUID) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

TopologyBuilder (org.apache.kafka.streams.processor.TopologyBuilder)38 Test (org.junit.Test)34 HashSet (java.util.HashSet)27 UUID (java.util.UUID)25 RandomUUIDFactory (org.apache.rya.api.function.projection.RandomUUIDFactory)25 VisibilityStatement (org.apache.rya.api.model.VisibilityStatement)24 TopologyFactory (org.apache.rya.streams.kafka.topology.TopologyFactory)24 ValueFactory (org.openrdf.model.ValueFactory)24 ValueFactoryImpl (org.openrdf.model.impl.ValueFactoryImpl)24 VisibilityBindingSet (org.apache.rya.api.model.VisibilityBindingSet)23 ArrayList (java.util.ArrayList)20 MapBindingSet (org.openrdf.query.impl.MapBindingSet)19 StreamsConfig (org.apache.kafka.streams.StreamsConfig)10 Metrics (org.apache.kafka.common.metrics.Metrics)8 MockClientSupplier (org.apache.kafka.test.MockClientSupplier)8 StreamsMetrics (org.apache.kafka.streams.StreamsMetrics)7 TaskId (org.apache.kafka.streams.processor.TaskId)7 TopicPartition (org.apache.kafka.common.TopicPartition)6 MockTime (org.apache.kafka.common.utils.MockTime)6 Properties (java.util.Properties)5