Examples with QueryChangeLog org.apache.rya.streams.api.queries.QueryChangeLog used on opensource projects

Example 16 with QueryChangeLog

use of org.apache.rya.streams.api.queries.QueryChangeLog in project incubator-rya by apache.

the class LogEventWorkerTest method notify_logDeleted_exists.

@Test
public void notify_logDeleted_exists() throws Exception {
    // The signal that will kill the working thread.
    final AtomicBoolean shutdownSignal = new AtomicBoolean(false);
    // The queue used to feed work.
    final BlockingQueue<LogEvent> logEventQueue = new ArrayBlockingQueue<>(10);
    // The queue work is written to.
    final BlockingQueue<QueryEvent> queryEventQueue = new ArrayBlockingQueue<>(10);
    // The Query Change Log that will be watched.
    final QueryChangeLog changeLog = new InMemoryQueryChangeLog();
    // Start the worker that will be tested.
    final Thread logEventWorker = new Thread(new LogEventWorker(logEventQueue, queryEventQueue, 50, TimeUnit.MILLISECONDS, shutdownSignal));
    logEventWorker.start();
    try {
        // Write a unit of work that indicates a log was created.
        final LogEvent createLogEvent = LogEvent.create("rya", changeLog);
        logEventQueue.offer(createLogEvent);
        // Write a unit of work that indicates a log was deleted.
        logEventQueue.offer(LogEvent.delete("rya"));
        // Show that a single unit of work was created for deleting everything for "rya".
        assertEquals(QueryEvent.stopALL("rya"), queryEventQueue.poll(500, TimeUnit.MILLISECONDS));
        assertNull(queryEventQueue.poll(500, TimeUnit.MILLISECONDS));
    } finally {
        shutdownSignal.set(true);
        logEventWorker.join();
    }
}

Example 17 with QueryChangeLog

use of org.apache.rya.streams.api.queries.QueryChangeLog in project incubator-rya by apache.

the class RunQueryCommand method execute.

@Override
public void execute(final String[] args) throws ArgumentsException, ExecutionException {
    requireNonNull(args);
    // Parse the command line arguments.
    final RunParameters params = new RunParameters();
    try {
        new JCommander(params, args);
    } catch (final ParameterException e) {
        throw new ArgumentsException("Could not add a new query because of invalid command line parameters.", e);
    }
    // Create the Kafka backed QueryChangeLog.
    final String bootstrapServers = params.kafkaIP + ":" + params.kafkaPort;
    final String topic = KafkaTopics.queryChangeLogTopic(params.ryaInstance);
    final QueryChangeLog queryChangeLog = KafkaQueryChangeLogFactory.make(bootstrapServers, topic);
    // The RunQuery command doesn't use the scheduled service feature.
    final Scheduler scheduler = Scheduler.newFixedRateSchedule(0L, 5, TimeUnit.SECONDS);
    final QueryRepository queryRepo = new InMemoryQueryRepository(queryChangeLog, scheduler);
    // Look up the query to be executed from the change log.
    try {
        try {
            final UUID queryId = UUID.fromString(params.queryId);
            final Optional<StreamsQuery> query = queryRepo.get(queryId);
            if (!query.isPresent()) {
                throw new ArgumentsException("There is no registered query for queryId " + params.queryId);
            }
            // Make sure the topics required by the application exists for the specified Rya instances.
            final Set<String> topics = new HashSet<>();
            topics.add(KafkaTopics.statementsTopic(params.ryaInstance));
            topics.add(KafkaTopics.queryResultsTopic(params.ryaInstance, queryId));
            KafkaTopics.createTopics(params.zookeeperServers, topics, 1, 1);
            // Run the query that uses those topics.
            final KafkaRunQuery runQuery = new KafkaRunQuery(params.kafkaIP, params.kafkaPort, KafkaTopics.statementsTopic(params.ryaInstance), KafkaTopics.queryResultsTopic(params.ryaInstance, queryId), queryRepo, new TopologyFactory());
            runQuery.run(queryId);
        } catch (final Exception e) {
            throw new ExecutionException("Could not execute the Run Query command.", e);
        }
    } catch (final ExecutionException e) {
        // Rethrow the exceptions that are advertised by execute.
        throw e;
    } catch (final Exception e) {
        throw new ExecutionException("Problem encountered while closing the QueryRepository.", e);
    }
}

Example 18 with QueryChangeLog

use of org.apache.rya.streams.api.queries.QueryChangeLog in project incubator-rya by apache.

the class DeleteQueryCommandIT method setup.

@Before
public void setup() {
    // Make sure the topic that the change log uses exists.
    final String changeLogTopic = KafkaTopics.queryChangeLogTopic(ryaInstance);
    System.out.println("Test Change Log Topic: " + changeLogTopic);
    kafka.createTopic(changeLogTopic);
    // Setup the QueryRepository used by the test.
    final Producer<?, QueryChange> queryProducer = KafkaTestUtil.makeProducer(kafka, StringSerializer.class, QueryChangeSerializer.class);
    final Consumer<?, QueryChange> queryConsumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, QueryChangeDeserializer.class);
    final QueryChangeLog changeLog = new KafkaQueryChangeLog(queryProducer, queryConsumer, changeLogTopic);
    queryRepo = new InMemoryQueryRepository(changeLog, Scheduler.newFixedRateSchedule(0L, 5, TimeUnit.SECONDS));
}

Example 19 with QueryChangeLog

use of org.apache.rya.streams.api.queries.QueryChangeLog in project incubator-rya by apache.

the class KafkaRyaStreamsClientFactory method make.

/**
 * Initialize a {@link RyaStreamsClient} that will interact with an instance of Rya Streams
 * that is backed by Kafka.
 *
 * @param ryaInstance - The name of the Rya Instance the client is connected to. (not null)
 * @param kafkaHostname - The hostname of the Kafka Broker.
 * @param kafkaPort - The port of the Kafka Broker.
 * @return The initialized commands.
 */
public static RyaStreamsClient make(final String ryaInstance, final String kafkaHostname, final int kafkaPort) {
    requireNonNull(ryaInstance);
    requireNonNull(kafkaHostname);
    // Setup Query Repository used by the Kafka Rya Streams subsystem.
    final Producer<?, QueryChange> queryProducer = makeProducer(kafkaHostname, kafkaPort, StringSerializer.class, QueryChangeSerializer.class);
    final Consumer<?, QueryChange> queryConsumer = fromStartConsumer(kafkaHostname, kafkaPort, StringDeserializer.class, QueryChangeDeserializer.class);
    final String changeLogTopic = KafkaTopics.queryChangeLogTopic(ryaInstance);
    final QueryChangeLog changeLog = new KafkaQueryChangeLog(queryProducer, queryConsumer, changeLogTopic);
    final QueryRepository queryRepo = new InMemoryQueryRepository(changeLog, SCHEDULER);
    // Create the Rya Streams client that is backed by a Kafka Query Change Log.
    return new RyaStreamsClient(new DefaultAddQuery(queryRepo), new DefaultGetQuery(queryRepo), new DefaultDeleteQuery(queryRepo), new KafkaGetQueryResultStream<>(kafkaHostname, "" + kafkaPort, VisibilityStatementDeserializer.class), new KafkaGetQueryResultStream<>(kafkaHostname, "" + kafkaPort, VisibilityBindingSetDeserializer.class), new DefaultListQueries(queryRepo), new DefaultStartQuery(queryRepo), new DefaultStopQuery(queryRepo)) {

        /**
         * Close the QueryRepository used by the returned client.
         */
        @Override
        public void close() {
            try {
                queryRepo.stopAndWait();
            } catch (final Exception e) {
                log.warn("Couldn't close a QueryRepository.", e);
            }
        }
    };
}

Example 20 with QueryChangeLog

use of org.apache.rya.streams.api.queries.QueryChangeLog in project incubator-rya by apache.

the class AddQueryCommand method execute.

@Override
public void execute(final String[] args) throws ArgumentsException, ExecutionException {
    requireNonNull(args);
    // Parse the command line arguments.
    final AddParameters params = new AddParameters();
    try {
        new JCommander(params, args);
    } catch (final ParameterException e) {
        throw new ArgumentsException("Could not add a new query because of invalid command line parameters.", e);
    }
    // Create the Kafka backed QueryChangeLog.
    final String bootstrapServers = params.kafkaIP + ":" + params.kafkaPort;
    final String topic = KafkaTopics.queryChangeLogTopic(params.ryaInstance);
    final QueryChangeLog queryChangeLog = KafkaQueryChangeLogFactory.make(bootstrapServers, topic);
    // The AddQuery command doesn't use the scheduled service feature.
    final Scheduler scheduler = Scheduler.newFixedRateSchedule(0L, 5, TimeUnit.SECONDS);
    final QueryRepository queryRepo = new InMemoryQueryRepository(queryChangeLog, scheduler);
    // Execute the add query command.
    try {
        final AddQuery addQuery = new DefaultAddQuery(queryRepo);
        try {
            final Boolean isActive = Boolean.parseBoolean(params.isActive);
            final Boolean isInsert = Boolean.parseBoolean(params.isInsert);
            // If the query's results are meant to be written back to Rya, make sure it creates statements.
            if (isInsert) {
                final boolean isConstructQuery = QueryInvestigator.isConstruct(params.query);
                final boolean isInsertQuery = QueryInvestigator.isInsertWhere(params.query);
                if (isConstructQuery) {
                    System.out.println("WARNING: CONSTRUCT is part of the SPARQL Query API, so they do not normally\n" + "get written back to the triple store. Consider using an INSERT, which is\n" + "part of the SPARQL Update API, in the future.");
                }
                if (!(isConstructQuery || isInsertQuery)) {
                    throw new ArgumentsException("Only CONSTRUCT queries and INSERT updates may be inserted back to the triple store.");
                }
            }
            final StreamsQuery query = addQuery.addQuery(params.query, isActive, isInsert);
            System.out.println("Added query: " + query.getSparql());
        } catch (final RyaStreamsException e) {
            throw new ExecutionException("Unable to add the query to Rya Streams.", e);
        }
    } catch (final MalformedQueryException e) {
        throw new ArgumentsException("Could not parse the provided query.", e);
    }
}