Examples with TopologyFactory org.apache.rya.streams.kafka.topology.TopologyFactory used on opensource projects

Example 11 with TopologyFactory

use of org.apache.rya.streams.kafka.topology.TopologyFactory in project incubator-rya by apache.

the class JoinProcessorIT method manyJoins.

@Test
public void manyJoins() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Setup a topology.
    final String query = "SELECT * WHERE { " + "?person <urn:talksTo> ?employee ." + "?employee <urn:worksAt> ?business ." + "?employee <urn:hourlyWage> ?wage ." + " }";
    final TopologyFactory factory = new TopologyFactory();
    final TopologyBuilder builder = factory.build(query, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Create some statements that generate a bunch of right SP results.
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:talksTo"), vf.createURI("urn:Bob")), "a"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:worksAt"), vf.createURI("urn:BurgerJoint")), "a"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:hourlyWage"), vf.createLiteral(7.25)), "a"));
    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("employee", vf.createURI("urn:Bob"));
    bs.addBinding("business", vf.createURI("urn:BurgerJoint"));
    bs.addBinding("wage", vf.createLiteral(7.25));
    expected.add(new VisibilityBindingSet(bs, "a"));
    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}

Example 12 with TopologyFactory

use of org.apache.rya.streams.kafka.topology.TopologyFactory in project incubator-rya by apache.

the class ProjectionProcessorIT method showProcessorWorks.

@Test
public void showProcessorWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Create a topology for the Query that will be tested.
    final String sparql = "SELECT (?person AS ?p) ?otherPerson " + "WHERE { " + "?person <urn:talksTo> ?otherPerson . " + "}";
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Load some data into the input topic.
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:talksTo"), vf.createURI("urn:Bob")), "a"));
    // Show the correct binding set results from the job.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet expectedBs = new MapBindingSet();
    expectedBs.addBinding("p", vf.createURI("urn:Alice"));
    expectedBs.addBinding("otherPerson", vf.createURI("urn:Bob"));
    expected.add(new VisibilityBindingSet(expectedBs, "a"));
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, Sets.newHashSet(expected), VisibilityBindingSetDeserializer.class);
}

Example 13 with TopologyFactory

use of org.apache.rya.streams.kafka.topology.TopologyFactory in project incubator-rya by apache.

the class RunQueryCommand method execute.

@Override
public void execute(final String[] args) throws ArgumentsException, ExecutionException {
    requireNonNull(args);
    // Parse the command line arguments.
    final RunParameters params = new RunParameters();
    try {
        new JCommander(params, args);
    } catch (final ParameterException e) {
        throw new ArgumentsException("Could not add a new query because of invalid command line parameters.", e);
    }
    // Create the Kafka backed QueryChangeLog.
    final String bootstrapServers = params.kafkaIP + ":" + params.kafkaPort;
    final String topic = KafkaTopics.queryChangeLogTopic(params.ryaInstance);
    final QueryChangeLog queryChangeLog = KafkaQueryChangeLogFactory.make(bootstrapServers, topic);
    // The RunQuery command doesn't use the scheduled service feature.
    final Scheduler scheduler = Scheduler.newFixedRateSchedule(0L, 5, TimeUnit.SECONDS);
    final QueryRepository queryRepo = new InMemoryQueryRepository(queryChangeLog, scheduler);
    // Look up the query to be executed from the change log.
    try {
        try {
            final UUID queryId = UUID.fromString(params.queryId);
            final Optional<StreamsQuery> query = queryRepo.get(queryId);
            if (!query.isPresent()) {
                throw new ArgumentsException("There is no registered query for queryId " + params.queryId);
            }
            // Make sure the topics required by the application exists for the specified Rya instances.
            final Set<String> topics = new HashSet<>();
            topics.add(KafkaTopics.statementsTopic(params.ryaInstance));
            topics.add(KafkaTopics.queryResultsTopic(params.ryaInstance, queryId));
            KafkaTopics.createTopics(params.zookeeperServers, topics, 1, 1);
            // Run the query that uses those topics.
            final KafkaRunQuery runQuery = new KafkaRunQuery(params.kafkaIP, params.kafkaPort, KafkaTopics.statementsTopic(params.ryaInstance), KafkaTopics.queryResultsTopic(params.ryaInstance, queryId), queryRepo, new TopologyFactory());
            runQuery.run(queryId);
        } catch (final Exception e) {
            throw new ExecutionException("Could not execute the Run Query command.", e);
        }
    } catch (final ExecutionException e) {
        // Rethrow the exceptions that are advertised by execute.
        throw e;
    } catch (final Exception e) {
        throw new ExecutionException("Problem encountered while closing the QueryRepository.", e);
    }
}

Example 14 with TopologyFactory

use of org.apache.rya.streams.kafka.topology.TopologyFactory in project incubator-rya by apache.

the class AggregationProcessorIT method multipleAggregations.

@Test
public void multipleAggregations() throws Exception {
    // A query that figures out what the youngest and oldest ages are across all people.
    final String sparql = "SELECT (min(?age) as ?youngest) (max(?age) as ?oldest) " + "WHERE { " + "?person <urn:age> ?age " + "}";
    // Create the statements that will be input into the query..
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:age"), vf.createLiteral(13)), ""));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:age"), vf.createLiteral(14)), ""));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Charlie"), vf.createURI("urn:age"), vf.createLiteral(7)), ""));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:David"), vf.createURI("urn:age"), vf.createLiteral(5)), ""));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Eve"), vf.createURI("urn:age"), vf.createLiteral(25)), ""));
    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("youngest", vf.createLiteral(13));
    bs.addBinding("oldest", vf.createLiteral(13));
    expected.add(new VisibilityBindingSet(bs, ""));
    bs = new MapBindingSet();
    bs.addBinding("youngest", vf.createLiteral(13));
    bs.addBinding("oldest", vf.createLiteral(14));
    expected.add(new VisibilityBindingSet(bs, ""));
    bs = new MapBindingSet();
    bs.addBinding("youngest", vf.createLiteral(7));
    bs.addBinding("oldest", vf.createLiteral(14));
    expected.add(new VisibilityBindingSet(bs, ""));
    bs = new MapBindingSet();
    bs.addBinding("youngest", vf.createLiteral(5));
    bs.addBinding("oldest", vf.createLiteral(14));
    expected.add(new VisibilityBindingSet(bs, ""));
    bs = new MapBindingSet();
    bs.addBinding("youngest", vf.createLiteral(5));
    bs.addBinding("oldest", vf.createLiteral(25));
    expected.add(new VisibilityBindingSet(bs, ""));
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}

Example 15 with TopologyFactory

use of org.apache.rya.streams.kafka.topology.TopologyFactory in project incubator-rya by apache.

the class AggregationProcessorIT method count.

@Test
public void count() throws Exception {
    // A query that figures out how many books each person has.
    final String sparql = "SELECT ?person (count(?book) as ?bookCount) " + "WHERE { " + "?person <urn:hasBook> ?book " + "} GROUP BY ?person";
    // Create the statements that will be input into the query..
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:hasBook"), vf.createLiteral("Book 1")), "a"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:hasBook"), vf.createLiteral("Book 1")), ""));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:hasBook"), vf.createLiteral("Book 2")), "b"));
    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("bookCount", vf.createLiteral("1", XMLSchema.INTEGER));
    expected.add(new VisibilityBindingSet(bs, "a"));
    bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Bob"));
    bs.addBinding("bookCount", vf.createLiteral("1", XMLSchema.INTEGER));
    expected.add(new VisibilityBindingSet(bs, ""));
    bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("bookCount", vf.createLiteral("2", XMLSchema.INTEGER));
    expected.add(new VisibilityBindingSet(bs, "a&b"));
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);
    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());
    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}