Examples with BindingSetRecord org.apache.rya.periodic.notification.api.BindingSetRecord used on opensource projects

Example 1 with BindingSetRecord

use of org.apache.rya.periodic.notification.api.BindingSetRecord in project incubator-rya by apache.

the class PeriodicNotificationProcessorIT method periodicProcessorTest.

@Test
public void periodicProcessorTest() throws Exception {
    String id = UUID.randomUUID().toString().replace("-", "");
    BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
    BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
    BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
    TimestampedNotification ts1 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
    long binId1 = (ts1.getTimestamp().getTime() / ts1.getPeriod()) * ts1.getPeriod();
    Thread.sleep(2000);
    TimestampedNotification ts2 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
    long binId2 = (ts2.getTimestamp().getTime() / ts2.getPeriod()) * ts2.getPeriod();
    Set<NodeBin> expectedBins = new HashSet<>();
    expectedBins.add(new NodeBin(id, binId1));
    expectedBins.add(new NodeBin(id, binId2));
    Set<BindingSet> expected = new HashSet<>();
    Set<VisibilityBindingSet> storageResults = new HashSet<>();
    QueryBindingSet bs1 = new QueryBindingSet();
    bs1.addBinding("periodicBinId", vf.createLiteral(binId1));
    bs1.addBinding("id", vf.createLiteral(1));
    expected.add(bs1);
    storageResults.add(new VisibilityBindingSet(bs1));
    QueryBindingSet bs2 = new QueryBindingSet();
    bs2.addBinding("periodicBinId", vf.createLiteral(binId1));
    bs2.addBinding("id", vf.createLiteral(2));
    expected.add(bs2);
    storageResults.add(new VisibilityBindingSet(bs2));
    QueryBindingSet bs3 = new QueryBindingSet();
    bs3.addBinding("periodicBinId", vf.createLiteral(binId2));
    bs3.addBinding("id", vf.createLiteral(3));
    expected.add(bs3);
    storageResults.add(new VisibilityBindingSet(bs3));
    QueryBindingSet bs4 = new QueryBindingSet();
    bs4.addBinding("periodicBinId", vf.createLiteral(binId2));
    bs4.addBinding("id", vf.createLiteral(4));
    expected.add(bs4);
    storageResults.add(new VisibilityBindingSet(bs4));
    PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), RYA_INSTANCE_NAME);
    periodicStorage.createPeriodicQuery(id, "select ?id where {?obs <urn:hasId> ?id.}", new VariableOrder("periodicBinId", "id"));
    periodicStorage.addPeriodicQueryResults(id, storageResults);
    NotificationProcessorExecutor processor = new NotificationProcessorExecutor(periodicStorage, notifications, bins, bindingSets, 1);
    processor.start();
    notifications.add(ts1);
    notifications.add(ts2);
    Thread.sleep(5000);
    Assert.assertEquals(expectedBins.size(), bins.size());
    Assert.assertEquals(true, bins.containsAll(expectedBins));
    Set<BindingSet> actual = new HashSet<>();
    bindingSets.forEach(x -> actual.add(x.getBindingSet()));
    Assert.assertEquals(expected, actual);
    processor.stop();
}

Example 2 with BindingSetRecord

use of org.apache.rya.periodic.notification.api.BindingSetRecord in project incubator-rya by apache.

the class PeriodicNotificationExporterIT method testExporter.

@Test
public void testExporter() throws InterruptedException {
    final String topic1 = kafkaTestInstanceRule.getKafkaTopicName() + "1";
    final String topic2 = kafkaTestInstanceRule.getKafkaTopicName() + "2";
    kafkaTestInstanceRule.createTopic(topic1);
    kafkaTestInstanceRule.createTopic(topic2);
    final BlockingQueue<BindingSetRecord> records = new LinkedBlockingQueue<>();
    final KafkaExporterExecutor exporter = new KafkaExporterExecutor(new KafkaProducer<String, BindingSet>(createKafkaProducerConfig()), 1, records);
    exporter.start();
    final QueryBindingSet bs1 = new QueryBindingSet();
    bs1.addBinding(PeriodicQueryResultStorage.PeriodicBinId, vf.createLiteral(1L));
    bs1.addBinding("name", vf.createURI("uri:Bob"));
    final BindingSetRecord record1 = new BindingSetRecord(bs1, topic1);
    final QueryBindingSet bs2 = new QueryBindingSet();
    bs2.addBinding(PeriodicQueryResultStorage.PeriodicBinId, vf.createLiteral(2L));
    bs2.addBinding("name", vf.createURI("uri:Joe"));
    final BindingSetRecord record2 = new BindingSetRecord(bs2, topic2);
    records.add(record1);
    records.add(record2);
    final Set<BindingSet> expected1 = new HashSet<>();
    expected1.add(bs1);
    final Set<BindingSet> expected2 = new HashSet<>();
    expected2.add(bs2);
    final Set<BindingSet> actual1 = getBindingSetsFromKafka(topic1);
    final Set<BindingSet> actual2 = getBindingSetsFromKafka(topic2);
    Assert.assertEquals(expected1, actual1);
    Assert.assertEquals(expected2, actual2);
    exporter.stop();
}

Example 3 with BindingSetRecord

use of org.apache.rya.periodic.notification.api.BindingSetRecord in project incubator-rya by apache.

the class PeriodicNotificationApplicationFactory method getPeriodicApplication.

/**
 * Create a PeriodicNotificationApplication.
 * @param conf - Configuration object that specifies the parameters needed to create the application
 * @return PeriodicNotificationApplication to periodically poll Rya Fluo for new results
 * @throws PeriodicApplicationException
 */
public static PeriodicNotificationApplication getPeriodicApplication(final PeriodicNotificationApplicationConfiguration conf) throws PeriodicApplicationException {
    final Properties kafkaConsumerProps = getKafkaConsumerProperties(conf);
    final Properties kafkaProducerProps = getKafkaProducerProperties(conf);
    final BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
    final BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
    final BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
    FluoClient fluo = null;
    try {
        final PeriodicQueryResultStorage storage = getPeriodicQueryResultStorage(conf);
        fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf);
        final NotificationCoordinatorExecutor coordinator = getCoordinator(conf.getCoordinatorThreads(), notifications);
        addRegisteredNotices(coordinator, fluo.newSnapshot());
        final KafkaExporterExecutor exporter = getExporter(conf.getExporterThreads(), kafkaProducerProps, bindingSets);
        final PeriodicQueryPrunerExecutor pruner = getPruner(storage, fluo, conf.getPrunerThreads(), bins);
        final NotificationProcessorExecutor processor = getProcessor(storage, notifications, bins, bindingSets, conf.getProcessorThreads());
        final KafkaNotificationProvider provider = getProvider(conf.getProducerThreads(), conf.getNotificationTopic(), coordinator, kafkaConsumerProps);
        return PeriodicNotificationApplication.builder().setCoordinator(coordinator).setProvider(provider).setExporter(exporter).setProcessor(processor).setPruner(pruner).build();
    } catch (AccumuloException | AccumuloSecurityException e) {
        throw new PeriodicApplicationException(e.getMessage());
    }
}

Example 4 with BindingSetRecord

use of org.apache.rya.periodic.notification.api.BindingSetRecord in project incubator-rya by apache.

the class TimestampedNotificationProcessor method processNotification.

/**
 * Processes the TimestampNotifications by scanning the PCJ tables for
 * entries in the bin corresponding to
 * {@link TimestampedNotification#getTimestamp()} and adding them to the
 * export BlockingQueue. The TimestampNotification is then used to form a
 * {@link NodeBin} that is passed to the BinPruner BlockingQueue so that the
 * bins can be deleted from Fluo and Accumulo.
 */
@Override
public void processNotification(final TimestampedNotification notification) {
    final String id = notification.getId();
    final long ts = notification.getTimestamp().getTime();
    final long period = notification.getPeriod();
    final long bin = getBinFromTimestamp(ts, period);
    final NodeBin nodeBin = new NodeBin(id, bin);
    try (CloseableIterator<BindingSet> iter = periodicStorage.listResults(id, Optional.of(bin))) {
        while (iter.hasNext()) {
            bindingSets.add(new BindingSetRecord(iter.next(), id));
        }
        // add NodeBin to BinPruner queue so that bin can be deleted from
        // Fluo and Accumulo
        bins.add(nodeBin);
    } catch (final Exception e) {
        log.warn("Encountered exception while accessing periodic results for bin: " + bin + " for query: " + id, e);
    }
}