Examples with KafkaQueryChangeLog org.apache.rya.streams.kafka.queries.KafkaQueryChangeLog used on opensource projects

Example 1 with KafkaQueryChangeLog

use of org.apache.rya.streams.kafka.queries.KafkaQueryChangeLog in project incubator-rya by apache.

the class KafkaQueryChangeLogSource method runOneIteration.

@Override
protected void runOneIteration() throws Exception {
    lock.lock();
    try {
        // Get the list of topics from the Kafka Server.
        final Set<String> changeLogTopics = new HashSet<>(listTopicsConsumer.listTopics().keySet());
        // Remove all topics that are not valid Rya Query Change Log topic names.
        changeLogTopics.removeIf(topic -> !KafkaTopics.getRyaInstanceFromQueryChangeLog(topic).isPresent());
        // Extract the Rya instance names from the change log topics.
        final Set<String> ryaInstances = changeLogTopics.stream().map(topic -> KafkaTopics.getRyaInstanceFromQueryChangeLog(topic).get()).collect(Collectors.toSet());
        // Any Rya instances that are in the old set of topics, but not the new one, have been deleted.
        final Set<String> deletedRyaInstances = new HashSet<>(Sets.difference(knownChangeLogs.keySet(), ryaInstances));
        // Any Rya instances that are in the new set of topics, but not the old set, have been created.
        final Set<String> createdRyaInstances = new HashSet<>(Sets.difference(ryaInstances, knownChangeLogs.keySet()));
        // Handle the deletes.
        for (final String deletedRyaInstance : deletedRyaInstances) {
            // Remove the change log from the set of known logs.
            knownChangeLogs.remove(deletedRyaInstance);
            // Notify the listeners of the update so that they may close the previously provided change log.
            for (final SourceListener listener : listeners) {
                listener.notifyDelete(deletedRyaInstance);
            }
        }
        // Handle the adds.
        for (final String createdRyaInstance : createdRyaInstances) {
            // Create and store the ChangeLog.
            final String changeLogTopic = KafkaTopics.queryChangeLogTopic(createdRyaInstance);
            knownChangeLogs.put(createdRyaInstance, changeLogTopic);
            // Notify the listeners of the update.
            for (final SourceListener listener : listeners) {
                final KafkaQueryChangeLog changeLog = KafkaQueryChangeLogFactory.make(kafkaBootstrapServer, changeLogTopic);
                listener.notifyCreate(createdRyaInstance, changeLog);
            }
        }
    } finally {
        lock.unlock();
    }
}

Example 2 with KafkaQueryChangeLog

use of org.apache.rya.streams.kafka.queries.KafkaQueryChangeLog in project incubator-rya by apache.

the class KafkaQueryChangeLogSource method subscribe.

@Override
public void subscribe(final SourceListener listener) {
    requireNonNull(listener);
    lock.lock();
    try {
        // Add the listener to the list of known listeners.
        listeners.add(listener);
        // Notify it with everything that already exists.
        for (final Entry<String, String> entry : knownChangeLogs.entrySet()) {
            final String changeLogTopic = entry.getValue();
            final KafkaQueryChangeLog changeLog = KafkaQueryChangeLogFactory.make(kafkaBootstrapServer, changeLogTopic);
            listener.notifyCreate(entry.getKey(), changeLog);
        }
    } finally {
        lock.unlock();
    }
}

Example 3 with KafkaQueryChangeLog

use of org.apache.rya.streams.kafka.queries.KafkaQueryChangeLog in project incubator-rya by apache.

the class RunQueryCommandIT method setup.

@Before
public void setup() {
    // Make sure the topic that the change log uses exists.
    final String changeLogTopic = KafkaTopics.queryChangeLogTopic("" + ryaInstance);
    kafka.createTopic(changeLogTopic);
    // Setup the QueryRepository used by the test.
    final Producer<?, QueryChange> queryProducer = KafkaTestUtil.makeProducer(kafka, StringSerializer.class, QueryChangeSerializer.class);
    final Consumer<?, QueryChange> queryConsumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, QueryChangeDeserializer.class);
    final QueryChangeLog changeLog = new KafkaQueryChangeLog(queryProducer, queryConsumer, changeLogTopic);
    queryRepo = new InMemoryQueryRepository(changeLog, Scheduler.newFixedRateSchedule(0L, 5, TimeUnit.SECONDS));
    // Initialize the Statements Producer and the Results Consumer.
    stmtProducer = KafkaTestUtil.makeProducer(kafka, StringSerializer.class, VisibilityStatementSerializer.class);
    resultConsumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, VisibilityBindingSetDeserializer.class);
}

Example 4 with KafkaQueryChangeLog

use of org.apache.rya.streams.kafka.queries.KafkaQueryChangeLog in project incubator-rya by apache.

the class AddQueryCommandIT method setup.

@Before
public void setup() {
    // Make sure the topic that the change log uses exists.
    final String changeLogTopic = KafkaTopics.queryChangeLogTopic("" + ryaInstance);
    kafka.createTopic(changeLogTopic);
    // Setup the QueryRepository used by the test.
    final Producer<?, QueryChange> queryProducer = KafkaTestUtil.makeProducer(kafka, StringSerializer.class, QueryChangeSerializer.class);
    final Consumer<?, QueryChange> queryConsumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, QueryChangeDeserializer.class);
    final QueryChangeLog changeLog = new KafkaQueryChangeLog(queryProducer, queryConsumer, changeLogTopic);
    queryRepo = new InMemoryQueryRepository(changeLog, Scheduler.newFixedRateSchedule(0L, 5, TimeUnit.SECONDS));
}

Example 5 with KafkaQueryChangeLog

use of org.apache.rya.streams.kafka.queries.KafkaQueryChangeLog in project incubator-rya by apache.

the class ListQueryCommandIT method setup.

@Before
public void setup() {
    // Make sure the topic that the change log uses exists.
    final String changeLogTopic = KafkaTopics.queryChangeLogTopic("" + ryaInstance);
    kafka.createTopic(changeLogTopic);
    // Setup the QueryRepository used by the test.
    final Producer<?, QueryChange> queryProducer = KafkaTestUtil.makeProducer(kafka, StringSerializer.class, QueryChangeSerializer.class);
    final Consumer<?, QueryChange> queryConsumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, QueryChangeDeserializer.class);
    final QueryChangeLog changeLog = new KafkaQueryChangeLog(queryProducer, queryConsumer, changeLogTopic);
    queryRepo = new InMemoryQueryRepository(changeLog, Scheduler.newFixedRateSchedule(0L, 5, TimeUnit.SECONDS));
}