Examples with AccumuloPeriodicQueryResultStorage org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage used on opensource projects

Example 6 with AccumuloPeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.

the class PeriodicNotificationApplicationFactory method getPeriodicQueryResultStorage.

private static PeriodicQueryResultStorage getPeriodicQueryResultStorage(final PeriodicNotificationApplicationConfiguration conf) throws AccumuloException, AccumuloSecurityException {
    final Instance instance = new ZooKeeperInstance(conf.getAccumuloInstance(), conf.getAccumuloZookeepers());
    final Connector conn = instance.getConnector(conf.getAccumuloUser(), new PasswordToken(conf.getAccumuloPassword()));
    final String ryaInstance = conf.getTablePrefix();
    return new AccumuloPeriodicQueryResultStorage(conn, ryaInstance);
}

Example 7 with AccumuloPeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.

the class CreateDeletePeriodicPCJ method runTest.

private void runTest(String query, Collection<Statement> statements, int expectedEntries) throws Exception {
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        String topic = "notification_topic";
        PeriodicQueryResultStorage storage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), RYA_INSTANCE_NAME);
        PeriodicNotificationClient notificationClient = new KafkaNotificationRegistrationClient(topic, getNotificationProducer("localhost:9092"));
        CreatePeriodicQuery periodicPCJ = new CreatePeriodicQuery(fluoClient, storage);
        String id = periodicPCJ.createPeriodicQuery(query, notificationClient).getQueryId();
        loadData(statements);
        // Ensure the data was loaded.
        final List<Bytes> rows = getFluoTableEntries(fluoClient);
        assertEquals(expectedEntries, rows.size());
        DeletePeriodicQuery deletePeriodic = new DeletePeriodicQuery(fluoClient, storage);
        deletePeriodic.deletePeriodicQuery(FluoQueryUtils.convertFluoQueryIdToPcjId(id), notificationClient);
        getMiniFluo().waitForObservers();
        // Ensure all data related to the query has been removed.
        final List<Bytes> empty_rows = getFluoTableEntries(fluoClient);
        assertEquals(1, empty_rows.size());
        // Ensure that Periodic Service notified to add and delete PeriodicNotification
        Set<CommandNotification> notifications;
        try (KafkaConsumer<String, CommandNotification> consumer = makeNotificationConsumer(topic)) {
            notifications = getKafkaNotifications(topic, 7000, consumer);
        }
        assertEquals(2, notifications.size());
        String notificationId = "";
        boolean addCalled = false;
        boolean deleteCalled = false;
        for (CommandNotification notification : notifications) {
            if (notificationId.length() == 0) {
                notificationId = notification.getId();
            } else {
                assertEquals(notificationId, notification.getId());
            }
            if (notification.getCommand() == Command.ADD) {
                addCalled = true;
            }
            if (notification.getCommand() == Command.DELETE) {
                deleteCalled = true;
            }
        }
        assertEquals(true, addCalled);
        assertEquals(true, deleteCalled);
    }
}

Example 8 with AccumuloPeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.

the class PeriodicBindingSetExporterFactory method build.

@Override
public Optional<IncrementalResultExporter> build(Context context) throws IncrementalExporterFactoryException, ConfigurationException {
    checkNotNull(context);
    // Wrap the context's parameters for parsing.
    final RyaExportParameters params = new RyaExportParameters(context.getObserverConfiguration().toMap());
    if (params.getUsePeriodicBindingSetExporter()) {
        // Setup Zookeeper connection info.
        final String accumuloInstance = params.getAccumuloInstanceName().get();
        final String zookeeperServers = params.getZookeeperServers().get().replaceAll(";", ",");
        final Instance inst = new ZooKeeperInstance(accumuloInstance, zookeeperServers);
        try {
            // Setup Accumulo connection info.
            final String exporterUsername = params.getExporterUsername().get();
            final String exporterPassword = params.getExporterPassword().get();
            final Connector accumuloConn = inst.getConnector(exporterUsername, new PasswordToken(exporterPassword));
            // Setup Rya PCJ Storage.
            final String ryaInstanceName = params.getRyaInstanceName().get();
            final PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(accumuloConn, ryaInstanceName);
            // Make the exporter.
            final IncrementalBindingSetExporter exporter = new PeriodicBindingSetExporter(periodicStorage);
            return Optional.of(exporter);
        } catch (final AccumuloException | AccumuloSecurityException e) {
            throw new IncrementalExporterFactoryException("Could not initialize the Accumulo connector using the provided configuration.", e);
        }
    } else {
        return Optional.absent();
    }
}

Example 9 with AccumuloPeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.

the class PeriodicNotificationBinPrunerIT method periodicPrunerTest.

@Test
public void periodicPrunerTest() throws Exception {
    String sparql = // n
    "prefix function: <http://org.apache.rya/function#> " + // n
    "prefix time: <http://www.w3.org/2006/time#> " + // n
    "select ?id (count(?obs) as ?total) where {" + // n
    "Filter(function:periodic(?time, 2, .5, time:hours)) " + // n
    "?obs <uri:hasTime> ?time. " + // n
    "?obs <uri:hasId> ?id } group by ?id";
    FluoClient fluo = new FluoClientImpl(super.getFluoConfiguration());
    // initialize resources and create pcj
    PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(super.getAccumuloConnector(), getRyaInstanceName());
    CreatePeriodicQuery createPeriodicQuery = new CreatePeriodicQuery(fluo, periodicStorage);
    String queryId = FluoQueryUtils.convertFluoQueryIdToPcjId(createPeriodicQuery.createPeriodicQuery(sparql).getQueryId());
    // create statements to ingest into Fluo
    final ValueFactory vf = new ValueFactoryImpl();
    final DatatypeFactory dtf = DatatypeFactory.newInstance();
    ZonedDateTime time = ZonedDateTime.now();
    long currentTime = time.toInstant().toEpochMilli();
    ZonedDateTime zTime1 = time.minusMinutes(30);
    String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT);
    ZonedDateTime zTime2 = zTime1.minusMinutes(30);
    String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT);
    ZonedDateTime zTime3 = zTime2.minusMinutes(30);
    String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT);
    ZonedDateTime zTime4 = zTime3.minusMinutes(30);
    String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT);
    final Collection<Statement> statements = Sets.newHashSet(vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time1))), vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time2))), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))), vf.createStatement(vf.createURI("urn:obs_3"), vf.createURI("uri:hasId"), vf.createLiteral("id_3")), vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time4))), vf.createStatement(vf.createURI("urn:obs_4"), vf.createURI("uri:hasId"), vf.createLiteral("id_4")), vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time4))), vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasId"), vf.createLiteral("id_1")), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasTime"), vf.createLiteral(dtf.newXMLGregorianCalendar(time3))), vf.createStatement(vf.createURI("urn:obs_2"), vf.createURI("uri:hasId"), vf.createLiteral("id_2")));
    // add statements to Fluo
    InsertTriples inserter = new InsertTriples();
    statements.forEach(x -> inserter.insert(fluo, RdfToRyaConversions.convertStatement(x)));
    super.getMiniFluo().waitForObservers();
    // FluoITHelper.printFluoTable(fluo);
    // Create the expected results of the SPARQL query once the PCJ has been
    // computed.
    final Set<BindingSet> expected1 = new HashSet<>();
    final Set<BindingSet> expected2 = new HashSet<>();
    final Set<BindingSet> expected3 = new HashSet<>();
    final Set<BindingSet> expected4 = new HashSet<>();
    long period = 1800000;
    long binId = (currentTime / period) * period;
    long bin1 = binId;
    long bin2 = binId + period;
    long bin3 = binId + 2 * period;
    long bin4 = binId + 3 * period;
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin1));
    expected1.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin1));
    expected1.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin1));
    expected1.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_4", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin1));
    expected1.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin2));
    expected2.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin2));
    expected2.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_3", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin2));
    expected2.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin3));
    expected3.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_2", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin3));
    expected3.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("total", vf.createLiteral("1", XMLSchema.INTEGER));
    bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING));
    bs.addBinding("periodicBinId", vf.createLiteral(bin4));
    expected4.add(bs);
    // make sure that expected and actual results align after ingest
    compareResults(periodicStorage, queryId, bin1, expected1);
    compareResults(periodicStorage, queryId, bin2, expected2);
    compareResults(periodicStorage, queryId, bin3, expected3);
    compareResults(periodicStorage, queryId, bin4, expected4);
    BlockingQueue<NodeBin> bins = new LinkedBlockingQueue<>();
    PeriodicQueryPrunerExecutor pruner = new PeriodicQueryPrunerExecutor(periodicStorage, fluo, 1, bins);
    pruner.start();
    bins.add(new NodeBin(queryId, bin1));
    bins.add(new NodeBin(queryId, bin2));
    bins.add(new NodeBin(queryId, bin3));
    bins.add(new NodeBin(queryId, bin4));
    Thread.sleep(10000);
    compareResults(periodicStorage, queryId, bin1, new HashSet<>());
    compareResults(periodicStorage, queryId, bin2, new HashSet<>());
    compareResults(periodicStorage, queryId, bin3, new HashSet<>());
    compareResults(periodicStorage, queryId, bin4, new HashSet<>());
    compareFluoCounts(fluo, queryId, bin1);
    compareFluoCounts(fluo, queryId, bin2);
    compareFluoCounts(fluo, queryId, bin3);
    compareFluoCounts(fluo, queryId, bin4);
    pruner.stop();
}

Example 10 with AccumuloPeriodicQueryResultStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPeriodicQueryResultStorage in project incubator-rya by apache.

the class QueryIT method runTest.

public void runTest(final String sparql, final Collection<Statement> statements, final Collection<BindingSet> expectedResults, final ExportStrategy strategy) throws Exception {
    requireNonNull(sparql);
    requireNonNull(statements);
    requireNonNull(expectedResults);
    // Register the PCJ with Rya.
    final Connector accumuloConn = super.getAccumuloConnector();
    final RyaClient ryaClient = AccumuloRyaClientFactory.build(createConnectionDetails(), accumuloConn);
    switch(strategy) {
        case RYA:
            ryaClient.getCreatePCJ().createPCJ(getRyaInstanceName(), sparql);
            addStatementsAndWait(statements);
            // Fetch the value that is stored within the PCJ table.
            try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName())) {
                final String pcjId = pcjStorage.listPcjs().get(0);
                final Set<BindingSet> results = Sets.newHashSet(pcjStorage.listResults(pcjId));
                // Ensure the result of the query matches the expected result.
                assertEquals(expectedResults, results);
            }
            break;
        case PERIODIC:
            final PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(accumuloConn, getRyaInstanceName());
            final String periodicId = periodicStorage.createPeriodicQuery(sparql);
            try (FluoClient fluo = new FluoClientImpl(super.getFluoConfiguration())) {
                new CreateFluoPcj().createPcj(periodicId, sparql, Sets.newHashSet(ExportStrategy.PERIODIC), fluo);
            }
            addStatementsAndWait(statements);
            final Set<BindingSet> results = Sets.newHashSet();
            try (CloseableIterator<BindingSet> resultIter = periodicStorage.listResults(periodicId, Optional.empty())) {
                while (resultIter.hasNext()) {
                    results.add(resultIter.next());
                }
            }
            assertEquals(expectedResults, results);
            break;
        default:
            throw new RuntimeException("Invalid export option");
    }
}