Search in sources :

Example 1 with SingleThreadKafkaStreamsFactory

use of org.apache.rya.streams.kafka.SingleThreadKafkaStreamsFactory in project incubator-rya by apache.

the class QueryManagerDaemon method init.

@Override
public void init(final DaemonContext context) throws DaemonInitException, Exception {
    requireNonNull(context);
    // Parse the command line arguments for the configuration file to use.
    final String[] args = context.getArguments();
    final DaemonParameters params = new DaemonParameters();
    try {
        new JCommander(params).parse(args);
    } catch (final ParameterException e) {
        throw new DaemonInitException("Unable to parse the command line arguments.", e);
    }
    final Path configFile = params.config != null ? Paths.get(params.config) : DEFAULT_CONFIGURATION_PATH;
    log.info("Loading the following configuration file: " + configFile);
    // Unmarshall the configuration file into an object.
    final QueryManagerConfig config;
    try (final InputStream stream = Files.newInputStream(configFile)) {
        config = QueryManagerConfigUnmarshaller.unmarshall(stream);
    } catch (final JAXBException | SAXException e) {
        throw new DaemonInitException("Unable to marshall the configuration XML file: " + configFile, e);
    }
    // Read the source polling period from the configuration.
    final QueryChanngeLogDiscoveryPeriod periodConfig = config.getPerformanceTunning().getQueryChanngeLogDiscoveryPeriod();
    final long period = periodConfig.getValue().longValue();
    final TimeUnit units = TimeUnit.valueOf(periodConfig.getUnits().toString());
    log.info("Query Change Log Polling Period: " + period + " " + units);
    final Scheduler scheduler = Scheduler.newFixedRateSchedule(0, period, units);
    // Initialize a QueryChangeLogSource.
    final Kafka kafka = config.getQueryChangeLogSource().getKafka();
    log.info("Kafka Source: " + kafka.getHostname() + ":" + kafka.getPort());
    final QueryChangeLogSource source = new KafkaQueryChangeLogSource(kafka.getHostname(), kafka.getPort(), scheduler);
    // Initialize a QueryExecutor.
    final String zookeeperServers = config.getQueryExecutor().getLocalKafkaStreams().getZookeepers();
    final KafkaStreamsFactory streamsFactory = new SingleThreadKafkaStreamsFactory(kafka.getHostname() + ":" + kafka.getPort());
    final QueryExecutor queryExecutor = new LocalQueryExecutor(new CreateKafkaTopic(zookeeperServers), streamsFactory);
    // Initialize the QueryManager using the configured resources.
    manager = new QueryManager(queryExecutor, source, period, units);
}
Also used : Scheduler(com.google.common.util.concurrent.AbstractScheduledService.Scheduler) Kafka(org.apache.rya.streams.querymanager.xml.Kafka) SAXException(org.xml.sax.SAXException) LocalQueryExecutor(org.apache.rya.streams.querymanager.kafka.LocalQueryExecutor) JCommander(com.beust.jcommander.JCommander) LocalQueryExecutor(org.apache.rya.streams.querymanager.kafka.LocalQueryExecutor) TimeUnit(java.util.concurrent.TimeUnit) ParameterException(com.beust.jcommander.ParameterException) Path(java.nio.file.Path) QueryChanngeLogDiscoveryPeriod(org.apache.rya.streams.querymanager.xml.QueryManagerConfig.PerformanceTunning.QueryChanngeLogDiscoveryPeriod) CreateKafkaTopic(org.apache.rya.streams.kafka.interactor.CreateKafkaTopic) InputStream(java.io.InputStream) KafkaQueryChangeLogSource(org.apache.rya.streams.querymanager.kafka.KafkaQueryChangeLogSource) JAXBException(javax.xml.bind.JAXBException) KafkaStreamsFactory(org.apache.rya.streams.kafka.KafkaStreamsFactory) SingleThreadKafkaStreamsFactory(org.apache.rya.streams.kafka.SingleThreadKafkaStreamsFactory) SingleThreadKafkaStreamsFactory(org.apache.rya.streams.kafka.SingleThreadKafkaStreamsFactory) KafkaQueryChangeLogSource(org.apache.rya.streams.querymanager.kafka.KafkaQueryChangeLogSource) DaemonInitException(org.apache.commons.daemon.DaemonInitException) QueryManagerConfig(org.apache.rya.streams.querymanager.xml.QueryManagerConfig)

Example 2 with SingleThreadKafkaStreamsFactory

use of org.apache.rya.streams.kafka.SingleThreadKafkaStreamsFactory in project incubator-rya by apache.

the class LocalQueryExecutorIT method runQuery.

@Test
public void runQuery() throws Exception {
    // Test values.
    final String ryaInstance = "rya";
    final StreamsQuery sQuery = new StreamsQuery(UUID.randomUUID(), "SELECT * WHERE { ?person <urn:worksAt> ?business . }", true, false);
    // Create the statements that will be loaded.
    final ValueFactory vf = new ValueFactoryImpl();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:worksAt"), vf.createURI("urn:BurgerJoint")), "a"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:worksAt"), vf.createURI("urn:TacoShop")), "a"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createURI("urn:Charlie"), vf.createURI("urn:worksAt"), vf.createURI("urn:TacoShop")), "a"));
    // Create the expected results.
    final List<VisibilityBindingSet> expected = new ArrayList<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Alice"));
    bs.addBinding("business", vf.createURI("urn:BurgerJoint"));
    expected.add(new VisibilityBindingSet(bs, "a"));
    bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Bob"));
    bs.addBinding("business", vf.createURI("urn:TacoShop"));
    expected.add(new VisibilityBindingSet(bs, "a"));
    bs = new MapBindingSet();
    bs.addBinding("person", vf.createURI("urn:Charlie"));
    bs.addBinding("business", vf.createURI("urn:TacoShop"));
    expected.add(new VisibilityBindingSet(bs, "a"));
    // Start the executor that will be tested.
    final CreateKafkaTopic createKafkaTopic = new CreateKafkaTopic(kafka.getZookeeperServers());
    final String kafkaServers = kafka.getKafkaHostname() + ":" + kafka.getKafkaPort();
    final KafkaStreamsFactory jobFactory = new SingleThreadKafkaStreamsFactory(kafkaServers);
    final QueryExecutor executor = new LocalQueryExecutor(createKafkaTopic, jobFactory);
    executor.startAndWait();
    try {
        // Start the query.
        executor.startQuery(ryaInstance, sQuery);
        // Wait for the program to start.
        Thread.sleep(5000);
        // Write some statements to the program.
        final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
        final LoadStatements loadStatements = new KafkaLoadStatements(statementsTopic, stmtProducer);
        loadStatements.fromCollection(statements);
        // Read the output of the streams program.
        final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, sQuery.getQueryId());
        resultConsumer.subscribe(Lists.newArrayList(resultsTopic));
        final List<VisibilityBindingSet> results = KafkaTestUtil.pollForResults(500, 6, 3, resultConsumer);
        assertEquals(expected, results);
    } finally {
        executor.stopAndWait();
    }
}
Also used : VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) StreamsQuery(org.apache.rya.streams.api.entity.StreamsQuery) CreateKafkaTopic(org.apache.rya.streams.kafka.interactor.CreateKafkaTopic) KafkaStreamsFactory(org.apache.rya.streams.kafka.KafkaStreamsFactory) SingleThreadKafkaStreamsFactory(org.apache.rya.streams.kafka.SingleThreadKafkaStreamsFactory) KafkaLoadStatements(org.apache.rya.streams.kafka.interactor.KafkaLoadStatements) LoadStatements(org.apache.rya.streams.api.interactor.LoadStatements) ValueFactoryImpl(org.openrdf.model.impl.ValueFactoryImpl) ArrayList(java.util.ArrayList) ValueFactory(org.openrdf.model.ValueFactory) VisibilityStatement(org.apache.rya.api.model.VisibilityStatement) SingleThreadKafkaStreamsFactory(org.apache.rya.streams.kafka.SingleThreadKafkaStreamsFactory) QueryExecutor(org.apache.rya.streams.querymanager.QueryExecutor) KafkaLoadStatements(org.apache.rya.streams.kafka.interactor.KafkaLoadStatements) MapBindingSet(org.openrdf.query.impl.MapBindingSet) Test(org.junit.Test)

Aggregations

KafkaStreamsFactory (org.apache.rya.streams.kafka.KafkaStreamsFactory)2 SingleThreadKafkaStreamsFactory (org.apache.rya.streams.kafka.SingleThreadKafkaStreamsFactory)2 CreateKafkaTopic (org.apache.rya.streams.kafka.interactor.CreateKafkaTopic)2 JCommander (com.beust.jcommander.JCommander)1 ParameterException (com.beust.jcommander.ParameterException)1 Scheduler (com.google.common.util.concurrent.AbstractScheduledService.Scheduler)1 InputStream (java.io.InputStream)1 Path (java.nio.file.Path)1 ArrayList (java.util.ArrayList)1 TimeUnit (java.util.concurrent.TimeUnit)1 JAXBException (javax.xml.bind.JAXBException)1 DaemonInitException (org.apache.commons.daemon.DaemonInitException)1 VisibilityBindingSet (org.apache.rya.api.model.VisibilityBindingSet)1 VisibilityStatement (org.apache.rya.api.model.VisibilityStatement)1 StreamsQuery (org.apache.rya.streams.api.entity.StreamsQuery)1 LoadStatements (org.apache.rya.streams.api.interactor.LoadStatements)1 KafkaLoadStatements (org.apache.rya.streams.kafka.interactor.KafkaLoadStatements)1 QueryExecutor (org.apache.rya.streams.querymanager.QueryExecutor)1 KafkaQueryChangeLogSource (org.apache.rya.streams.querymanager.kafka.KafkaQueryChangeLogSource)1 LocalQueryExecutor (org.apache.rya.streams.querymanager.kafka.LocalQueryExecutor)1