use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.
the class PeriodicNotificationProcessorIT method periodicProcessorTest.
@Test
public void periodicProcessorTest() throws Exception {
String id = UUID.randomUUID().toString().replace("-", "");
BlockingQueue<TimestampedNotification> notifications = new LinkedBlockingQueue<>();
BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
BlockingQueue<BindingSetRecord> bindingSets = new LinkedBlockingQueue<>();
TimestampedNotification ts1 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
long binId1 = (ts1.getTimestamp().getTime() / ts1.getPeriod()) * ts1.getPeriod();
Thread.sleep(2000);
TimestampedNotification ts2 = new TimestampedNotification(PeriodicNotification.builder().id(id).initialDelay(0).period(2000).timeUnit(TimeUnit.SECONDS).build());
long binId2 = (ts2.getTimestamp().getTime() / ts2.getPeriod()) * ts2.getPeriod();
Set<NodeBin> expectedBins = new HashSet<>();
expectedBins.add(new NodeBin(id, binId1));
expectedBins.add(new NodeBin(id, binId2));
Set<BindingSet> expected = new HashSet<>();
Set<VisibilityBindingSet> storageResults = new HashSet<>();
QueryBindingSet bs1 = new QueryBindingSet();
bs1.addBinding("periodicBinId", vf.createLiteral(binId1));
bs1.addBinding("id", vf.createLiteral(1));
expected.add(bs1);
storageResults.add(new VisibilityBindingSet(bs1));
QueryBindingSet bs2 = new QueryBindingSet();
bs2.addBinding("periodicBinId", vf.createLiteral(binId1));
bs2.addBinding("id", vf.createLiteral(2));
expected.add(bs2);
storageResults.add(new VisibilityBindingSet(bs2));
QueryBindingSet bs3 = new QueryBindingSet();
bs3.addBinding("periodicBinId", vf.createLiteral(binId2));
bs3.addBinding("id", vf.createLiteral(3));
expected.add(bs3);
storageResults.add(new VisibilityBindingSet(bs3));
QueryBindingSet bs4 = new QueryBindingSet();
bs4.addBinding("periodicBinId", vf.createLiteral(binId2));
bs4.addBinding("id", vf.createLiteral(4));
expected.add(bs4);
storageResults.add(new VisibilityBindingSet(bs4));
PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), RYA_INSTANCE_NAME);
periodicStorage.createPeriodicQuery(id, "select ?id where {?obs <urn:hasId> ?id.}", new VariableOrder("periodicBinId", "id"));
periodicStorage.addPeriodicQueryResults(id, storageResults);
NotificationProcessorExecutor processor = new NotificationProcessorExecutor(periodicStorage, notifications, bins, bindingSets, 1);
processor.start();
notifications.add(ts1);
notifications.add(ts2);
Thread.sleep(5000);
Assert.assertEquals(expectedBins.size(), bins.size());
Assert.assertEquals(true, bins.containsAll(expectedBins));
Set<BindingSet> actual = new HashSet<>();
bindingSets.forEach(x -> actual.add(x.getBindingSet()));
Assert.assertEquals(expected, actual);
processor.stop();
}
use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.
the class AccumuloPeriodicQueryResultStorageIT method init.
@Before
public void init() throws AccumuloException, AccumuloSecurityException {
super.getConnector().securityOperations().changeUserAuthorizations("root", new Authorizations("U"));
periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getConnector(), RYA);
}
use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.
the class PeriodicNotificationApplicationIT method periodicApplicationWithAggTest.
@Test
public void periodicApplicationWithAggTest() throws Exception {
final String sparql = // n
"prefix function: <http://org.apache.rya/function#> " + // n
"prefix time: <http://www.w3.org/2006/time#> " + // n
"select (count(?obs) as ?total) where {" + // n
"Filter(function:periodic(?time, 1, .25, time:minutes)) " + // n
"?obs <uri:hasTime> ?time. " + // n
"?obs <uri:hasId> ?id } ";
// make data
final int periodMult = 15;
final ValueFactory vf = new ValueFactoryImpl();
final DatatypeFactory dtf = DatatypeFactory.newInstance();
// results more predictable
while (System.currentTimeMillis() % (periodMult * 1000) > 500) {
;
}
final ZonedDateTime time = ZonedDateTime.now();
final ZonedDateTime zTime1 = time.minusSeconds(2 * periodMult);
final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
final ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
final ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
final Collection<Statement> statements = Sets.newHashSet(vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))), vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")));
try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
final Connector connector = ConfigUtils.getConnector(conf);
final PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
final CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
final String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
addData(statements);
app.start();
final Multimap<Long, BindingSet> expected = HashMultimap.create();
try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
consumer.subscribe(Arrays.asList(id));
final long end = System.currentTimeMillis() + 4 * periodMult * 1000;
long lastBinId = 0L;
long binId = 0L;
final List<Long> ids = new ArrayList<>();
while (System.currentTimeMillis() < end) {
final ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult * 1000);
for (final ConsumerRecord<String, BindingSet> record : records) {
final BindingSet result = record.value();
binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
if (lastBinId != binId) {
lastBinId = binId;
ids.add(binId);
}
expected.put(binId, result);
}
}
Assert.assertEquals(3, expected.asMap().size());
int i = 0;
for (final Long ident : ids) {
Assert.assertEquals(1, expected.get(ident).size());
final BindingSet bs = expected.get(ident).iterator().next();
final Value val = bs.getValue("total");
final int total = Integer.parseInt(val.stringValue());
Assert.assertEquals(3 - i, total);
i++;
}
}
final Set<BindingSet> expectedResults = new HashSet<>();
try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
results.forEachRemaining(x -> expectedResults.add(x));
Assert.assertEquals(0, expectedResults.size());
}
}
}
use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.
the class PeriodicNotificationApplicationIT method periodicApplicationTest.
@Test
public void periodicApplicationTest() throws Exception {
final String sparql = // n
"prefix function: <http://org.apache.rya/function#> " + // n
"prefix time: <http://www.w3.org/2006/time#> " + // n
"select ?obs ?id where {" + // n
"Filter(function:periodic(?time, 1, .25, time:minutes)) " + // n
"?obs <uri:hasTime> ?time. " + // n
"?obs <uri:hasId> ?id } ";
// make data
final int periodMult = 15;
final ValueFactory vf = new ValueFactoryImpl();
final DatatypeFactory dtf = DatatypeFactory.newInstance();
// results more predictable
while (System.currentTimeMillis() % (periodMult * 1000) > 500) {
;
}
final ZonedDateTime time = ZonedDateTime.now();
final ZonedDateTime zTime1 = time.minusSeconds(2 * periodMult);
final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
final ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
final ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
final Collection<Statement> statements = Sets.newHashSet(vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))), vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")));
try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
final Connector connector = ConfigUtils.getConnector(conf);
final PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
final CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
final String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
addData(statements);
app.start();
final Multimap<Long, BindingSet> expected = HashMultimap.create();
try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
consumer.subscribe(Arrays.asList(id));
final long end = System.currentTimeMillis() + 4 * periodMult * 1000;
long lastBinId = 0L;
long binId = 0L;
final List<Long> ids = new ArrayList<>();
while (System.currentTimeMillis() < end) {
final ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult * 1000);
for (final ConsumerRecord<String, BindingSet> record : records) {
final BindingSet result = record.value();
binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
if (lastBinId != binId) {
lastBinId = binId;
ids.add(binId);
}
expected.put(binId, result);
}
}
Assert.assertEquals(3, expected.asMap().size());
int i = 0;
for (final Long ident : ids) {
Assert.assertEquals(3 - i, expected.get(ident).size());
i++;
}
}
final Set<BindingSet> expectedResults = new HashSet<>();
try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
results.forEachRemaining(x -> expectedResults.add(x));
Assert.assertEquals(0, expectedResults.size());
}
}
}
use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.
the class PeriodicNotificationApplicationIT method periodicApplicationWithAggAndGroupByTest.
@Test
public void periodicApplicationWithAggAndGroupByTest() throws Exception {
final String sparql = // n
"prefix function: <http://org.apache.rya/function#> " + // n
"prefix time: <http://www.w3.org/2006/time#> " + // n
"select ?type (count(?obs) as ?total) where {" + // n
"Filter(function:periodic(?time, 1, .25, time:minutes)) " + // n
"?obs <uri:hasTime> ?time. " + // n
"?obs <uri:hasObsType> ?type } group by ?type";
// make data
final int periodMult = 15;
final ValueFactory vf = new ValueFactoryImpl();
final DatatypeFactory dtf = DatatypeFactory.newInstance();
// results more predictable
while (System.currentTimeMillis() % (periodMult * 1000) > 500) {
;
}
final ZonedDateTime time = ZonedDateTime.now();
final ZonedDateTime zTime1 = time.minusSeconds(2 * periodMult);
final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
final ZonedDateTime zTime2 = zTime1.minusSeconds(periodMult);
final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
final ZonedDateTime zTime3 = zTime2.minusSeconds(periodMult);
final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
final Collection<Statement> statements = Sets.newHashSet(vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))), vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasObsType"), vf.createLiteral("ship")), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasObsType"), vf.createLiteral("airplane")), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasObsType"), vf.createLiteral("ship")), vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))), vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasObsType"), vf.createLiteral("airplane")), vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))), vf.createStatement(vf.createURI("urn:obs_5"), vf.createURI("uri:hasObsType"), vf.createLiteral("automobile")));
try (FluoClient fluo = FluoClientFactory.getFluoClient(conf.getFluoAppName(), Optional.of(conf.getFluoTableName()), conf)) {
final Connector connector = ConfigUtils.getConnector(conf);
final PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(connector, conf.getTablePrefix());
final CreatePeriodicQuery periodicQuery = new CreatePeriodicQuery(fluo, storage);
final String id = FluoQueryUtils.convertFluoQueryIdToPcjId(periodicQuery.createPeriodicQuery(sparql, registrar).getQueryId());
addData(statements);
app.start();
final Multimap<Long, BindingSet> actual = HashMultimap.create();
try (KafkaConsumer<String, BindingSet> consumer = new KafkaConsumer<>(kafkaProps, new StringDeserializer(), new BindingSetSerDe())) {
consumer.subscribe(Arrays.asList(id));
final long end = System.currentTimeMillis() + 4 * periodMult * 1000;
long lastBinId = 0L;
long binId = 0L;
final List<Long> ids = new ArrayList<>();
while (System.currentTimeMillis() < end) {
final ConsumerRecords<String, BindingSet> records = consumer.poll(periodMult * 1000);
for (final ConsumerRecord<String, BindingSet> record : records) {
final BindingSet result = record.value();
binId = Long.parseLong(result.getBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID).getValue().stringValue());
if (lastBinId != binId) {
lastBinId = binId;
ids.add(binId);
}
actual.put(binId, result);
}
}
final Map<Long, Set<BindingSet>> expected = new HashMap<>();
final Set<BindingSet> expected1 = new HashSet<>();
final QueryBindingSet bs1 = new QueryBindingSet();
bs1.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
bs1.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
bs1.addBinding("type", vf.createLiteral("airplane"));
final QueryBindingSet bs2 = new QueryBindingSet();
bs2.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
bs2.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
bs2.addBinding("type", vf.createLiteral("ship"));
final QueryBindingSet bs3 = new QueryBindingSet();
bs3.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(0)));
bs3.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
bs3.addBinding("type", vf.createLiteral("automobile"));
expected1.add(bs1);
expected1.add(bs2);
expected1.add(bs3);
final Set<BindingSet> expected2 = new HashSet<>();
final QueryBindingSet bs4 = new QueryBindingSet();
bs4.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(1)));
bs4.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
bs4.addBinding("type", vf.createLiteral("airplane"));
final QueryBindingSet bs5 = new QueryBindingSet();
bs5.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(1)));
bs5.addBinding("total", new LiteralImpl("2", XMLSchema.INTEGER));
bs5.addBinding("type", vf.createLiteral("ship"));
expected2.add(bs4);
expected2.add(bs5);
final Set<BindingSet> expected3 = new HashSet<>();
final QueryBindingSet bs6 = new QueryBindingSet();
bs6.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(2)));
bs6.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
bs6.addBinding("type", vf.createLiteral("ship"));
final QueryBindingSet bs7 = new QueryBindingSet();
bs7.addBinding(IncrementalUpdateConstants.PERIODIC_BIN_ID, vf.createLiteral(ids.get(2)));
bs7.addBinding("total", new LiteralImpl("1", XMLSchema.INTEGER));
bs7.addBinding("type", vf.createLiteral("airplane"));
expected3.add(bs6);
expected3.add(bs7);
expected.put(ids.get(0), expected1);
expected.put(ids.get(1), expected2);
expected.put(ids.get(2), expected3);
Assert.assertEquals(3, actual.asMap().size());
for (final Long ident : ids) {
Assert.assertEquals(expected.get(ident), actual.get(ident));
}
}
final Set<BindingSet> expectedResults = new HashSet<>();
try (CloseableIterator<BindingSet> results = storage.listResults(id, Optional.empty())) {
results.forEachRemaining(x -> expectedResults.add(x));
Assert.assertEquals(0, expectedResults.size());
}
}
}
Aggregations