use of org.apache.rya.periodic.notification.notification.TimestampedNotification in project incubator-rya by apache.
the class PeriodicNotificationProcessorIT method periodicProcessorTest.
@Test
public void periodicProcessorTest() throws Exception {
String id = UUID.randomUUID().toString().replace("-", "");
BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
TimestampedNotification ts1 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
long binId1 = (ts1.getTimestamp().getTime() / ts1.getPeriod()) * ts1.getPeriod();
Thread.sleep(2000);
TimestampedNotification ts2 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
long binId2 = (ts2.getTimestamp().getTime() / ts2.getPeriod()) * ts2.getPeriod();
Set<NodeBin> expectedBins = new HashSet<>();
expectedBins.add(new NodeBin(id, binId1));
expectedBins.add(new NodeBin(id, binId2));
Set<BindingSet> expected = new HashSet<>();
Set<VisibilityBindingSet> storageResults = new HashSet<>();
QueryBindingSet bs1 = new QueryBindingSet();
bs1.addBinding("periodicBinId", vf.createLiteral(binId1));
bs1.addBinding("id", vf.createLiteral(1));
expected.add(bs1);
storageResults.add(new VisibilityBindingSet(bs1));
QueryBindingSet bs2 = new QueryBindingSet();
bs2.addBinding("periodicBinId", vf.createLiteral(binId1));
bs2.addBinding("id", vf.createLiteral(2));
expected.add(bs2);
storageResults.add(new VisibilityBindingSet(bs2));
QueryBindingSet bs3 = new QueryBindingSet();
bs3.addBinding("periodicBinId", vf.createLiteral(binId2));
bs3.addBinding("id", vf.createLiteral(3));
expected.add(bs3);
storageResults.add(new VisibilityBindingSet(bs3));
QueryBindingSet bs4 = new QueryBindingSet();
bs4.addBinding("periodicBinId", vf.createLiteral(binId2));
bs4.addBinding("id", vf.createLiteral(4));
expected.add(bs4);
storageResults.add(new VisibilityBindingSet(bs4));
PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), RYA_INSTANCE_NAME);
periodicStorage.createPeriodicQuery(id, "select ?id where {?obs <urn:hasId> ?id.}", new VariableOrder("periodicBinId", "id"));
periodicStorage.addPeriodicQueryResults(id, storageResults);
NotificationProcessorExecutor processor = new NotificationProcessorExecutor(periodicStorage, notifications, bins, bindingSets, 1);
processor.start();
notifications.add(ts1);
notifications.add(ts2);
Thread.sleep(5000);
Assert.assertEquals(expectedBins.size(), bins.size());
Assert.assertEquals(true, bins.containsAll(expectedBins));
Set<BindingSet> actual = new HashSet<>();
bindingSets.forEach(x -> actual.add(x.getBindingSet()));
Assert.assertEquals(expected, actual);
processor.stop();
}
use of org.apache.rya.periodic.notification.notification.TimestampedNotification in project incubator-rya by apache.
the class PeriodicNotificationApplicationFactory method getPeriodicApplication.
/**
* Create a PeriodicNotificationApplication.
* @param conf - Configuration object that specifies the parameters needed to create the application
* @return PeriodicNotificationApplication to periodically poll Rya Fluo for new results
* @throws PeriodicApplicationException
*/
public static PeriodicNotificationApplication getPeriodicApplication(final PeriodicNotificationApplicationConfiguration conf) throws PeriodicApplicationException {
final Properties kafkaConsumerProps = getKafkaConsumerProperties(conf);
final Properties kafkaProducerProps = getKafkaProducerProperties(conf);
final BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
final BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
final BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
FluoClient fluo = null;
try {
final PeriodicQueryResultStorage storage = getPeriodicQueryResultStorage(conf);
fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf);
final NotificationCoordinatorExecutor coordinator = getCoordinator(conf.getCoordinatorThreads(), notifications);
addRegisteredNotices(coordinator, fluo.newSnapshot());
final KafkaExporterExecutor exporter = getExporter(conf.getExporterThreads(), kafkaProducerProps, bindingSets);
final PeriodicQueryPrunerExecutor pruner = getPruner(storage, fluo, conf.getPrunerThreads(), bins);
final NotificationProcessorExecutor processor = getProcessor(storage, notifications, bins, bindingSets, conf.getProcessorThreads());
final KafkaNotificationProvider provider = getProvider(conf.getProducerThreads(), conf.getNotificationTopic(), coordinator, kafkaConsumerProps);
return PeriodicNotificationApplication.builder().setCoordinator(coordinator).setProvider(provider).setExporter(exporter).setProcessor(processor).setPruner(pruner).build();
} catch (AccumuloException | AccumuloSecurityException e) {
throw new PeriodicApplicationException(e.getMessage());
}
}
use of org.apache.rya.periodic.notification.notification.TimestampedNotification in project incubator-rya by apache.
the class PeriodicNotificationProviderIT method testProvider.
@Test
public void testProvider() throws MalformedQueryException, InterruptedException, UnsupportedQueryException {
String sparql = // n
"prefix function: <http://org.apache.rya/function#> " + // n
"prefix time: <http://www.w3.org/2006/time#> " + // n
"select ?id (count(?obs) as ?total) where {" + // n
"Filter(function:periodic(?time, 1, .25, time:minutes)) " + // n
"?obs <uri:hasTime> ?time. " + // n
"?obs <uri:hasId> ?id } group by ?id";
BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
PeriodicNotificationCoordinatorExecutor coord = new PeriodicNotificationCoordinatorExecutor(2, notifications);
PeriodicNotificationProvider provider = new PeriodicNotificationProvider();
CreateFluoPcj pcj = new CreateFluoPcj();
String id = null;
try (FluoClient fluo = new FluoClientImpl(getFluoConfiguration())) {
id = pcj.createPcj(FluoQueryUtils.createNewPcjId(), sparql, Sets.newHashSet(), fluo).getQueryId();
provider.processRegisteredNotifications(coord, fluo.newSnapshot());
}
TimestampedNotification notification = notifications.take();
Assert.assertEquals(5000, notification.getInitialDelay());
Assert.assertEquals(15000, notification.getPeriod());
Assert.assertEquals(TimeUnit.MILLISECONDS, notification.getTimeUnit());
Assert.assertEquals(FluoQueryUtils.convertFluoQueryIdToPcjId(id), notification.getId());
}
use of org.apache.rya.periodic.notification.notification.TimestampedNotification in project incubator-rya by apache.
the class PeriodicCommandNotificationConsumerIT method kafkaNotificationMillisProviderTest.
@Test
public void kafkaNotificationMillisProviderTest() throws InterruptedException {
BasicConfigurator.configure();
final BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
final Properties props = createKafkaConfig();
final KafkaProducer<String, CommandNotification> producer = new KafkaProducer<>(props);
final String topic = rule.getKafkaTopicName();
rule.createTopic(topic);
registration = new KafkaNotificationRegistrationClient(topic, producer);
coord = new PeriodicNotificationCoordinatorExecutor(1, notifications);
provider = new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord, 1);
provider.start();
registration.addNotification("1", 1000, 0, TimeUnit.MILLISECONDS);
Thread.sleep(4000);
// check that notifications are being added to the blocking queue
Assert.assertEquals(true, notifications.size() > 0);
registration.deleteNotification("1");
Thread.sleep(2000);
final int size = notifications.size();
// sleep for 2 seconds to ensure no more messages being produced
Thread.sleep(2000);
Assert.assertEquals(size, notifications.size());
tearDown();
}
use of org.apache.rya.periodic.notification.notification.TimestampedNotification in project incubator-rya by apache.
the class PeriodicCommandNotificationConsumerIT method kafkaNotificationProviderTest.
@Test
public void kafkaNotificationProviderTest() throws InterruptedException {
BasicConfigurator.configure();
final BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
final Properties props = createKafkaConfig();
final KafkaProducer<String, CommandNotification> producer = new KafkaProducer<>(props);
final String topic = rule.getKafkaTopicName();
rule.createTopic(topic);
registration = new KafkaNotificationRegistrationClient(topic, producer);
coord = new PeriodicNotificationCoordinatorExecutor(1, notifications);
provider = new KafkaNotificationProvider(topic, new StringDeserializer(), new CommandNotificationSerializer(), props, coord, 1);
provider.start();
registration.addNotification("1", 1, 0, TimeUnit.SECONDS);
Thread.sleep(4000);
// check that notifications are being added to the blocking queue
Assert.assertEquals(true, notifications.size() > 0);
registration.deleteNotification("1");
Thread.sleep(2000);
final int size = notifications.size();
// sleep for 2 seconds to ensure no more messages being produced
Thread.sleep(2000);
Assert.assertEquals(size, notifications.size());
tearDown();
}
Aggregations