Examples with PeriodicQueryResultStorage org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage used on opensource projects

Example 1 with PeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage in project incubator-rya by apache.

the class PeriodicNotificationProcessorIT method periodicProcessorTest.

@Test
public void periodicProcessorTest() throws Exception {
    String id = UUID.randomUUID().toString().replace("-", "");
    BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
    BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
    BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
    TimestampedNotification ts1 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
    long binId1 = (ts1.getTimestamp().getTime() / ts1.getPeriod()) * ts1.getPeriod();
    Thread.sleep(2000);
    TimestampedNotification ts2 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
    long binId2 = (ts2.getTimestamp().getTime() / ts2.getPeriod()) * ts2.getPeriod();
    Set<NodeBin> expectedBins = new HashSet<>();
    expectedBins.add(new NodeBin(id, binId1));
    expectedBins.add(new NodeBin(id, binId2));
    Set<BindingSet> expected = new HashSet<>();
    Set<VisibilityBindingSet> storageResults = new HashSet<>();
    QueryBindingSet bs1 = new QueryBindingSet();
    bs1.addBinding("periodicBinId", vf.createLiteral(binId1));
    bs1.addBinding("id", vf.createLiteral(1));
    expected.add(bs1);
    storageResults.add(new VisibilityBindingSet(bs1));
    QueryBindingSet bs2 = new QueryBindingSet();
    bs2.addBinding("periodicBinId", vf.createLiteral(binId1));
    bs2.addBinding("id", vf.createLiteral(2));
    expected.add(bs2);
    storageResults.add(new VisibilityBindingSet(bs2));
    QueryBindingSet bs3 = new QueryBindingSet();
    bs3.addBinding("periodicBinId", vf.createLiteral(binId2));
    bs3.addBinding("id", vf.createLiteral(3));
    expected.add(bs3);
    storageResults.add(new VisibilityBindingSet(bs3));
    QueryBindingSet bs4 = new QueryBindingSet();
    bs4.addBinding("periodicBinId", vf.createLiteral(binId2));
    bs4.addBinding("id", vf.createLiteral(4));
    expected.add(bs4);
    storageResults.add(new VisibilityBindingSet(bs4));
    PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), RYA_INSTANCE_NAME);
    periodicStorage.createPeriodicQuery(id, "select ?id where {?obs <urn:hasId> ?id.}", new VariableOrder("periodicBinId", "id"));
    periodicStorage.addPeriodicQueryResults(id, storageResults);
    NotificationProcessorExecutor processor = new NotificationProcessorExecutor(periodicStorage, notifications, bins, bindingSets, 1);
    processor.start();
    notifications.add(ts1);
    notifications.add(ts2);
    Thread.sleep(5000);
    Assert.assertEquals(expectedBins.size(), bins.size());
    Assert.assertEquals(true, bins.containsAll(expectedBins));
    Set<BindingSet> actual = new HashSet<>();
    bindingSets.forEach(x -> actual.add(x.getBindingSet()));
    Assert.assertEquals(expected, actual);
    processor.stop();
}

Example 2 with PeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage in project incubator-rya by apache.

the class PeriodicNotificationApplicationIT method periodicApplicationWithAggTest.

@Test
public void periodicApplicationWithAggTest() throws Exception {
    final String sparql = // n
    "prefix function: <http://org.apache.rya/function#> " + // n
    "prefix time: <http://www.w3.org/2006/time#> " + // n
    "select (count(?obs) as ?total) where {" + // n
    "Filter(function:periodic(?time, 1, .25, time:minutes)) " + // n
    "?obs <uri:hasTime> ?time. " + // n
    "?obs <uri:hasId> ?id } ";
    // make data
    final int periodMult = 15;
    final ValueFactory vf = new ValueFactoryImpl();
    final DatatypeFactory dtf = DatatypeFactory.newInstance();
    // results more predictable
    while (System.currentTimeMillis() % (periodMult * 1000) > 500) {
        ;
    }
    final ZonedDateTime time = ZonedDateTime.now();
    final ZonedDateTime zTime1 = time.minusSeconds(2 * periodMult);
    final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
    final ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
    final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
    final ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
    final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
    final Collection<Statement> statements = Sets.newHashSet(vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))), vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")));
    try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
        final Connector connector = ConfigUtils.getConnector(conf);
        final PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
        final CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
        final String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
        addData(statements);
        app.start();
        final Multimap<Long, BindingSet> expected = HashMultimap.create();
        try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
            consumer.subscribe(Arrays.asList(id));
            final long end = System.currentTimeMillis() + 4 * periodMult * 1000;
            long lastBinId = 0L;
            long binId = 0L;
            final List<Long> ids = new ArrayList<>();
            while (System.currentTimeMillis() < end) {
                final ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult * 1000);
                for (final ConsumerRecord<String, BindingSet> record : records) {
                    final BindingSet result = record.value();
                    binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
                    if (lastBinId != binId) {
                        lastBinId = binId;
                        ids.add(binId);
                    }
                    expected.put(binId, result);
                }
            }
            Assert.assertEquals(3, expected.asMap().size());
            int i = 0;
            for (final Long ident : ids) {
                Assert.assertEquals(1, expected.get(ident).size());
                final BindingSet bs = expected.get(ident).iterator().next();
                final Value val = bs.getValue("total");
                final int total = Integer.parseInt(val.stringValue());
                Assert.assertEquals(3 - i, total);
                i++;
            }
        }
        final Set<BindingSet> expectedResults = new HashSet<>();
        try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
            results.forEachRemaining(x -> expectedResults.add(x));
            Assert.assertEquals(0, expectedResults.size());
        }
    }
}

Example 3 with PeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage in project incubator-rya by apache.

the class PeriodicNotificationApplicationIT method periodicApplicationTest.

@Test
public void periodicApplicationTest() throws Exception {
    final String sparql = // n
    "prefix function: <http://org.apache.rya/function#> " + // n
    "prefix time: <http://www.w3.org/2006/time#> " + // n
    "select ?obs ?id where {" + // n
    "Filter(function:periodic(?time, 1, .25, time:minutes)) " + // n
    "?obs <uri:hasTime> ?time. " + // n
    "?obs <uri:hasId> ?id } ";
    // make data
    final int periodMult = 15;
    final ValueFactory vf = new ValueFactoryImpl();
    final DatatypeFactory dtf = DatatypeFactory.newInstance();
    // results more predictable
    while (System.currentTimeMillis() % (periodMult * 1000) > 500) {
        ;
    }
    final ZonedDateTime time = ZonedDateTime.now();
    final ZonedDateTime zTime1 = time.minusSeconds(2 * periodMult);
    final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
    final ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
    final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
    final ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
    final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
    final Collection<Statement> statements = Sets.newHashSet(vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))), vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")));
    try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
        final Connector connector = ConfigUtils.getConnector(conf);
        final PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
        final CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
        final String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
        addData(statements);
        app.start();
        final Multimap<Long, BindingSet> expected = HashMultimap.create();
        try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
            consumer.subscribe(Arrays.asList(id));
            final long end = System.currentTimeMillis() + 4 * periodMult * 1000;
            long lastBinId = 0L;
            long binId = 0L;
            final List<Long> ids = new ArrayList<>();
            while (System.currentTimeMillis() < end) {
                final ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult * 1000);
                for (final ConsumerRecord<String, BindingSet> record : records) {
                    final BindingSet result = record.value();
                    binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
                    if (lastBinId != binId) {
                        lastBinId = binId;
                        ids.add(binId);
                    }
                    expected.put(binId, result);
                }
            }
            Assert.assertEquals(3, expected.asMap().size());
            int i = 0;
            for (final Long ident : ids) {
                Assert.assertEquals(3 - i, expected.get(ident).size());
                i++;
            }
        }
        final Set<BindingSet> expectedResults = new HashSet<>();
        try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
            results.forEachRemaining(x -> expectedResults.add(x));
            Assert.assertEquals(0, expectedResults.size());
        }
    }
}

Example 4 with PeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage in project incubator-rya by apache.

the class PeriodicNotificationApplicationIT method periodicApplicationWithAggAndGroupByTest.

@Test
public void periodicApplicationWithAggAndGroupByTest() throws Exception {
    final String sparql = // n
    "prefix function: <http://org.apache.rya/function#> " + // n
    "prefix time: <http://www.w3.org/2006/time#> " + // n
    "select ?type (count(?obs) as ?total) where {" + // n
    "Filter(function:periodic(?time, 1, .25, time:minutes)) " + // n
    "?obs <uri:hasTime> ?time. " + // n
    "?obs <uri:hasObsType> ?type } group by ?type";
    // make data
    final int periodMult = 15;
    final ValueFactory vf = new ValueFactoryImpl();
    final DatatypeFactory dtf = DatatypeFactory.newInstance();
    // results more predictable
    while (System.currentTimeMillis() % (periodMult * 1000) > 500) {
        ;
    }
    final ZonedDateTime time = ZonedDateTime.now();
    final ZonedDateTime zTime1 = time.minusSeconds(2 * periodMult);
    final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
    final ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
    final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
    final ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
    final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
    final Collection<Statement> statements = Sets.newHashSet(vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))), vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasObsType"), vf.createLiteral("ship")), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasObsType"), vf.createLiteral("airplane")), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasObsType"), vf.createLiteral("ship")), vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))), vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasObsType"), vf.createLiteral("airplane")), vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))), vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasObsType"), vf.createLiteral("automobile")));
    try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
        final Connector connector = ConfigUtils.getConnector(conf);
        final PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
        final CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
        final String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
        addData(statements);
        app.start();
        final Multimap<Long, BindingSet> actual = HashMultimap.create();
        try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
            consumer.subscribe(Arrays.asList(id));
            final long end = System.currentTimeMillis() + 4 * periodMult * 1000;
            long lastBinId = 0L;
            long binId = 0L;
            final List<Long> ids = new ArrayList<>();
            while (System.currentTimeMillis() < end) {
                final ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult * 1000);
                for (final ConsumerRecord<String, BindingSet> record : records) {
                    final BindingSet result = record.value();
                    binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
                    if (lastBinId != binId) {
                        lastBinId = binId;
                        ids.add(binId);
                    }
                    actual.put(binId, result);
                }
            }
            final Map<Long, Set<BindingSet>> expected = new HashMap<>();
            final Set<BindingSet> expected1 = new HashSet<>();
            final QueryBindingSet bs1 = new QueryBindingSet();
            bs1.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
            bs1.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
            bs1.addBinding("type", vf.createLiteral("airplane"));
            final QueryBindingSet bs2 = new QueryBindingSet();
            bs2.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
            bs2.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
            bs2.addBinding("type", vf.createLiteral("ship"));
            final QueryBindingSet bs3 = new QueryBindingSet();
            bs3.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
            bs3.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
            bs3.addBinding("type", vf.createLiteral("automobile"));
            expected1.add(bs1);
            expected1.add(bs2);
            expected1.add(bs3);
            final Set<BindingSet> expected2 = new HashSet<>();
            final QueryBindingSet bs4 = new QueryBindingSet();
            bs4.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(1)));
            bs4.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
            bs4.addBinding("type", vf.createLiteral("airplane"));
            final QueryBindingSet bs5 = new QueryBindingSet();
            bs5.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(1)));
            bs5.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
            bs5.addBinding("type", vf.createLiteral("ship"));
            expected2.add(bs4);
            expected2.add(bs5);
            final Set<BindingSet> expected3 = new HashSet<>();
            final QueryBindingSet bs6 = new QueryBindingSet();
            bs6.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(2)));
            bs6.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
            bs6.addBinding("type", vf.createLiteral("ship"));
            final QueryBindingSet bs7 = new QueryBindingSet();
            bs7.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(2)));
            bs7.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
            bs7.addBinding("type", vf.createLiteral("airplane"));
            expected3.add(bs6);
            expected3.add(bs7);
            expected.put(ids.get(0), expected1);
            expected.put(ids.get(1), expected2);
            expected.put(ids.get(2), expected3);
            Assert.assertEquals(3, actual.asMap().size());
            for (final Long ident : ids) {
                Assert.assertEquals(expected.get(ident), actual.get(ident));
            }
        }
        final Set<BindingSet> expectedResults = new HashSet<>();
        try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
            results.forEachRemaining(x -> expectedResults.add(x));
            Assert.assertEquals(0, expectedResults.size());
        }
    }
}

Example 5 with PeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.PeriodicQueryResultStorage in project incubator-rya by apache.

the class CreateDeletePeriodicPCJ method runTest.

private void runTest(String query, Collection<Statement> statements, int expectedEntries) throws Exception {
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        String topic = "notification_topic";
        PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), RYA_INSTANCE_NAME);
        PeriodicNotificationClient notificationClient = new KafkaNotificationRegistrationClient(topic, getNotificationProducer("localhost:9092"));
        CreatePeriodicQuery periodicPCJ = new CreatePeriodicQuery(fluoClient, storage);
        String id = periodicPCJ.createPeriodicQuery(query, notificationClient).getQueryId();
        loadData(statements);
        // Ensure the data was loaded.
        final List<Bytes> rows = getFluoTableEntries(fluoClient);
        assertEquals(expectedEntries, rows.size());
        DeletePeriodicQuery deletePeriodic = new DeletePeriodicQuery(fluoClient, storage);
        deletePeriodic.deletePeriodicQuery(FluoQueryUtils.convertFluoQueryIdToPcjId(id), notificationClient);
        getMiniFluo().waitForObservers();
        // Ensure all data related to the query has been removed.
        final List<Bytes> empty_rows = getFluoTableEntries(fluoClient);
        assertEquals(1, empty_rows.size());
        // Ensure that Periodic Service notified to add and delete PeriodicNotification
        Set<CommandNotification> notifications;
        try (KafkaConsumer<String, CommandNotification> consumer = makeNotificationConsumer(topic)) {
            notifications = getKafkaNotifications(topic, 7000, consumer);
        }
        assertEquals(2, notifications.size());
        String notificationId = "";
        boolean addCalled = false;
        boolean deleteCalled = false;
        for (CommandNotification notification : notifications) {
            if (notificationId.length() == 0) {
                notificationId = notification.getId();
            } else {
                assertEquals(notificationId, notification.getId());
            }
            if (notification.getCommand() == Command.ADD) {
                addCalled = true;
            }
            if (notification.getCommand() == Command.DELETE) {
                deleteCalled = true;
            }
        }
        assertEquals(true, addCalled);
        assertEquals(true, deleteCalled);
    }
}