Examples with CreateFluoPcj org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj used on opensource projects

Example 1 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class InputIT method historicThenStreamedResults.

/**
 * Simulates the case where a Triple is added to Rya, a new query that includes
 * that triple as a historic match is inserted into Fluo, and then some new
 * triple that matches the query is streamed into Fluo. The query's results
 * must include both the historic result and the newly streamed result.
 */
@Test
public void historicThenStreamedResults() throws Exception {
    // A query that finds people who talk to Eve and work at Chipotle.
    final String sparql = "SELECT ?x WHERE { " + "?x <http://talksTo> <http://Eve>. " + "?x <http://worksAt> <http://Chipotle>." + "}";
    // Triples that are loaded into Rya before the PCJ is created.
    final ValueFactory vf = new ValueFactoryImpl();
    final Set<Statement> historicTriples = Sets.newHashSet(vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")), vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")));
    // Triples that will be streamed into Fluo after the PCJ has been created.
    final Set<RyaStatement> streamedTriples = Sets.newHashSet(new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://talksTo"), new RyaURI("http://Eve")), new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")));
    // Load the historic data into Rya.
    final SailRepositoryConnection ryaConn = super.getRyaSailRepository().getConnection();
    for (final Statement triple : historicTriples) {
        ryaConn.add(triple);
    }
    ryaConn.close();
    // Create the PCJ table.
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = pcjStorage.createPcj(sparql);
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Tell the Fluo app to maintain the PCJ.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Ensure Alice is a match.
        super.getMiniFluo().waitForObservers();
        final Set<BindingSet> expected = new HashSet<>();
        MapBindingSet bs = new MapBindingSet();
        bs.addBinding("x", vf.createURI("http://Alice"));
        expected.add(bs);
        Set<BindingSet> results = new HashSet<>();
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            while (resultsIt.hasNext()) {
                results.add(resultsIt.next());
            }
        }
        assertEquals(expected, results);
        // Stream the data into Fluo.
        new InsertTriples().insert(fluoClient, streamedTriples, Optional.<String>absent());
        // Verify the end results of the query also include Frank.
        super.getMiniFluo().waitForObservers();
        bs = new MapBindingSet();
        bs.addBinding("x", vf.createURI("http://Frank"));
        expected.add(bs);
        results = new HashSet<>();
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            while (resultsIt.hasNext()) {
                results.add(resultsIt.next());
            }
        }
        assertEquals(expected, results);
    }
}

Example 2 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class InputIT method streamedResults.

/**
 * Ensure streamed matches are included in the result.
 */
@Test
public void streamedResults() throws Exception {
    // A query that finds people who talk to Eve and work at Chipotle.
    final String sparql = "SELECT ?x WHERE { " + "?x <http://talksTo> <http://Eve>. " + "?x <http://worksAt> <http://Chipotle>." + "}";
    // Triples that will be streamed into Fluo after the PCJ has been created.
    final Set<RyaStatement> streamedTriples = Sets.newHashSet(new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Eve")), new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://talksTo"), new RyaURI("http://Eve")), new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://talksTo"), new RyaURI("http://Eve")), new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://helps"), new RyaURI("http://Kevin")), new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), new RyaStatement(new RyaURI("http://David"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")));
    // The expected results of the SPARQL query once the PCJ has been computed.
    final ValueFactory vf = new ValueFactoryImpl();
    final Set<BindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("x", vf.createURI("http://Bob"));
    expected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("x", vf.createURI("http://Charlie"));
    expected.add(bs);
    // Create the PCJ table.
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = pcjStorage.createPcj(sparql);
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Tell the Fluo app to maintain the PCJ.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Ensure the query has no results yet.
        super.getMiniFluo().waitForObservers();
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            assertFalse(resultsIt.hasNext());
        }
        // Stream the data into Fluo.
        new InsertTriples().insert(fluoClient, streamedTriples, Optional.<String>absent());
        // Verify the end results of the query match the expected results.
        super.getMiniFluo().waitForObservers();
        final HashSet<BindingSet> results = new HashSet<>();
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            while (resultsIt.hasNext()) {
                results.add(resultsIt.next());
            }
        }
        assertEquals(expected, results);
    }
}

Example 3 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class KafkaRyaSubGraphExportIT method loadRyaStatements.

protected String loadRyaStatements(final String sparql, final Collection<RyaStatement> statements) throws Exception {
    requireNonNull(sparql);
    requireNonNull(statements);
    FluoClient client = null;
    try {
        CreateFluoPcj createPcj = new CreateFluoPcj();
        client = new FluoClientImpl(super.getFluoConfiguration());
        String id = createPcj.createPcj(sparql, client).getQueryId();
        AccumuloRyaDAO dao = getRyaDAO();
        dao.add(statements.iterator());
        // Wait for the Fluo application to finish computing the end result.
        super.getMiniFluo().waitForObservers();
        // FluoITHelper.printFluoTable(client);
        return id;
    } finally {
        if (client != null) {
            client.close();
        }
    }
}

Example 4 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class RyaExportIT method resultsExported.

@Test
public void resultsExported() throws Exception {
    final String sparql = "SELECT ?customer ?worker ?city " + "{ " + "FILTER(?customer = <http://Alice>) " + "FILTER(?city = <http://London>) " + "?customer <http://talksTo> ?worker. " + "?worker <http://livesIn> ?city. " + "?worker <http://worksAt> <http://Chipotle>. " + "}";
    // Triples that will be streamed into Fluo after the PCJ has been created.
    final ValueFactory vf = new ValueFactoryImpl();
    final Set<RyaStatement> streamedTriples = Sets.newHashSet(new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Bob")), new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://livesIn"), new RyaURI("http://London")), new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Charlie")), new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://livesIn"), new RyaURI("http://London")), new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://David")), new RyaStatement(new RyaURI("http://David"), new RyaURI("http://livesIn"), new RyaURI("http://London")), new RyaStatement(new RyaURI("http://David"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Eve")), new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://livesIn"), new RyaURI("http://Leeds")), new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://talksTo"), new RyaURI("http://Alice")), new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://livesIn"), new RyaURI("http://London")), new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")));
    // The expected results of the SPARQL query once the PCJ has been computed.
    final Set<BindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("customer", vf.createURI("http://Alice"));
    bs.addBinding("worker", vf.createURI("http://Bob"));
    bs.addBinding("city", vf.createURI("http://London"));
    expected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", vf.createURI("http://Alice"));
    bs.addBinding("worker", vf.createURI("http://Charlie"));
    bs.addBinding("city", vf.createURI("http://London"));
    expected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", vf.createURI("http://Alice"));
    bs.addBinding("worker", vf.createURI("http://David"));
    bs.addBinding("city", vf.createURI("http://London"));
    expected.add(bs);
    // Create the PCJ table.
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = pcjStorage.createPcj(sparql);
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Tell the Fluo app to maintain the PCJ.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Stream the data into Fluo.
        new InsertTriples().insert(fluoClient, streamedTriples, Optional.<String>absent());
        // Fetch the exported results from Accumulo once the observers finish working.
        super.getMiniFluo().waitForObservers();
        // Fetch expected results from the PCJ table that is in Accumulo.
        final Set<BindingSet> results = Sets.newHashSet(pcjStorage.listResults(pcjId));
        // Verify the end results of the query match the expected results.
        assertEquals(expected, results);
    }
}

Example 5 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class HistoricStreamingVisibilityIT method historicResults.

/**
 * Ensure historic matches are included in the result.
 */
@Test
public void historicResults() throws Exception {
    // A query that finds people who talk to Eve and work at Chipotle.
    final String sparql = "SELECT ?x " + "WHERE { " + "?x <http://talksTo> <http://Eve>. " + "?x <http://worksAt> <http://Chipotle>." + "}";
    final Connector accumuloConn = super.getAccumuloConnector();
    accumuloConn.securityOperations().changeUserAuthorizations(getUsername(), new Authorizations("U", "V", "W"));
    final AccumuloRyaDAO dao = new AccumuloRyaDAO();
    dao.setConnector(accumuloConn);
    dao.setConf(makeConfig());
    dao.init();
    // Triples that are loaded into Rya before the PCJ is created.
    final ValueFactory vf = new ValueFactoryImpl();
    final Set<RyaStatement> historicTriples = Sets.newHashSet(makeRyaStatement(vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")), "U"), makeRyaStatement(vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")), "V"), makeRyaStatement(vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")), "W"), makeRyaStatement(vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://helps"), vf.createURI("http://Kevin")), "U"), makeRyaStatement(vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")), "W"), makeRyaStatement(vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")), "V"), makeRyaStatement(vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")), "U"), makeRyaStatement(vf.createStatement(vf.createURI("http://David"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")), "V"));
    dao.add(historicTriples.iterator());
    dao.flush();
    // The expected results of the SPARQL query once the PCJ has been computed.
    final Set<BindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("x", vf.createURI("http://Bob"));
    expected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("x", vf.createURI("http://Charlie"));
    expected.add(bs);
    // Create the PCJ table.
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = pcjStorage.createPcj(sparql);
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
    }
    // Verify the end results of the query match the expected results.
    super.getMiniFluo().waitForObservers();
    final Set<BindingSet> results = Sets.newHashSet(pcjStorage.listResults(pcjId));
    Assert.assertEquals(expected, results);
}