Examples with KafkaNotificationRegistrationClient org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient used on opensource projects

Example 1 with KafkaNotificationRegistrationClient

use of org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient in project incubator-rya by apache.

the class PeriodicNotificationApplicationIT method init.

@Before
public void init() throws Exception {
    final String topic = rule.getKafkaTopicName();
    rule.createTopic(topic);
    // get user specified props and update with the embedded kafka bootstrap servers and rule generated topic
    props = getProps();
    props.setProperty(NOTIFICATION_TOPIC, topic);
    props.setProperty(KAFKA_BOOTSTRAP_SERVERS, bootstrapServers);
    conf = new PeriodicNotificationApplicationConfiguration(props);
    // create Kafka Producer
    kafkaProps = getKafkaProperties(conf);
    producer = new KafkaProducer<>(kafkaProps, new StringSerializer(), new CommandNotificationSerializer());
    // extract kafka specific properties from application config
    app = PeriodicNotificationApplicationFactory.getPeriodicApplication(conf);
    registrar = new KafkaNotificationRegistrationClient(conf.getNotificationTopic(), producer);
}

Example 2 with KafkaNotificationRegistrationClient

use of org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient in project incubator-rya by apache.

the class CreateDeletePeriodicPCJ method runTest.

private void runTest(String query, Collection<Statement> statements, int expectedEntries) throws Exception {
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        String topic = "notification_topic";
        PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), RYA_INSTANCE_NAME);
        PeriodicNotificationClient notificationClient = new KafkaNotificationRegistrationClient(topic, getNotificationProducer("localhost:9092"));
        CreatePeriodicQuery periodicPCJ = new CreatePeriodicQuery(fluoClient, storage);
        String id = periodicPCJ.createPeriodicQuery(query, notificationClient).getQueryId();
        loadData(statements);
        // Ensure the data was loaded.
        final List<Bytes> rows = getFluoTableEntries(fluoClient);
        assertEquals(expectedEntries, rows.size());
        DeletePeriodicQuery deletePeriodic = new DeletePeriodicQuery(fluoClient, storage);
        deletePeriodic.deletePeriodicQuery(FluoQueryUtils.convertFluoQueryIdToPcjId(id), notificationClient);
        getMiniFluo().waitForObservers();
        // Ensure all data related to the query has been removed.
        final List<Bytes> empty_rows = getFluoTableEntries(fluoClient);
        assertEquals(1, empty_rows.size());
        // Ensure that Periodic Service notified to add and delete PeriodicNotification
        Set<CommandNotification> notifications;
        try (KafkaConsumer<String, CommandNotification> consumer = makeNotificationConsumer(topic)) {
            notifications = getKafkaNotifications(topic, 7000, consumer);
        }
        assertEquals(2, notifications.size());
        String notificationId = "";
        boolean addCalled = false;
        boolean deleteCalled = false;
        for (CommandNotification notification : notifications) {
            if (notificationId.length() == 0) {
                notificationId = notification.getId();
            } else {
                assertEquals(notificationId, notification.getId());
            }
            if (notification.getCommand() == Command.ADD) {
                addCalled = true;
            }
            if (notification.getCommand() == Command.DELETE) {
                deleteCalled = true;
            }
        }
        assertEquals(true, addCalled);
        assertEquals(true, deleteCalled);
    }
}

Example 3 with KafkaNotificationRegistrationClient

use of org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient in project incubator-rya by apache.

the class PeriodicCommandNotificationConsumerIT method kafkaNotificationMillisProviderTest.

@Test
public void kafkaNotificationMillisProviderTest() throws InterruptedException {
    BasicConfigurator.configure();
    final BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
    final Properties props = createKafkaConfig();
    final KafkaProducer<String, CommandNotification> producer = new KafkaProducer<>(props);
    final String topic = rule.getKafkaTopicName();
    rule.createTopic(topic);
    registration = new KafkaNotificationRegistrationClient(topic, producer);
    coord = new PeriodicNotificationCoordinatorExecutor(1, notifications);
    provider = new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord, 1);
    provider.start();
    registration.addNotification("1", 1000, 0, TimeUnit.MILLISECONDS);
    Thread.sleep(4000);
    // check that notifications are being added to the blocking queue
    Assert.assertEquals(true, notifications.size() > 0);
    registration.deleteNotification("1");
    Thread.sleep(2000);
    final int size = notifications.size();
    // sleep for 2 seconds to ensure no more messages being produced
    Thread.sleep(2000);
    Assert.assertEquals(size, notifications.size());
    tearDown();
}

Example 4 with KafkaNotificationRegistrationClient

use of org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient in project incubator-rya by apache.

the class PeriodicCommandNotificationConsumerIT method kafkaNotificationProviderTest.

@Test
public void kafkaNotificationProviderTest() throws InterruptedException {
    BasicConfigurator.configure();
    final BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
    final Properties props = createKafkaConfig();
    final KafkaProducer<String, CommandNotification> producer = new KafkaProducer<>(props);
    final String topic = rule.getKafkaTopicName();
    rule.createTopic(topic);
    registration = new KafkaNotificationRegistrationClient(topic, producer);
    coord = new PeriodicNotificationCoordinatorExecutor(1, notifications);
    provider = new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord, 1);
    provider.start();
    registration.addNotification("1", 1, 0, TimeUnit.SECONDS);
    Thread.sleep(4000);
    // check that notifications are being added to the blocking queue
    Assert.assertEquals(true, notifications.size() > 0);
    registration.deleteNotification("1");
    Thread.sleep(2000);
    final int size = notifications.size();
    // sleep for 2 seconds to ensure no more messages being produced
    Thread.sleep(2000);
    Assert.assertEquals(size, notifications.size());
    tearDown();
}

Example 5 with KafkaNotificationRegistrationClient

use of org.apache.rya.periodic.notification.registration.KafkaNotificationRegistrationClient in project incubator-rya by apache.

the class AccumuloCreatePeriodicPCJ method updateFluoAppAndRegisterWithKafka.

private String updateFluoAppAndRegisterWithKafka(final String ryaInstance, final String fluoAppName, String sparql, String periodicTopic, String bootStrapServers) throws RepositoryException, MalformedQueryException, SailException, QueryEvaluationException, PcjException, RyaDAOException, UnsupportedQueryException, PeriodicQueryCreationException {
    requireNonNull(sparql);
    requireNonNull(periodicTopic);
    requireNonNull(bootStrapServers);
    final PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(getConnector(), ryaInstance);
    // Connect to the Fluo application that is updating this instance's PCJs.
    final AccumuloConnectionDetails cd = super.getAccumuloConnectionDetails();
    try (final FluoClient fluoClient = new FluoClientFactory().connect(cd.getUsername(), new String(cd.getUserPass()), cd.getInstanceName(), cd.getZookeepers(), fluoAppName)) {
        // Initialize the PCJ within the Fluo application.
        final CreatePeriodicQuery periodicPcj = new CreatePeriodicQuery(fluoClient, periodicStorage);
        PeriodicNotificationClient periodicClient = new KafkaNotificationRegistrationClient(periodicTopic, createProducer(bootStrapServers));
        return periodicPcj.withRyaIntegration(sparql, periodicClient, getConnector(), ryaInstance).getQueryId();
    }
}