Examples with PrecomputedJoinStorage org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage used on opensource projects

Example 26 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class FluoAndHistoricPcjsDemo method execute.

@Override
public void execute(final MiniAccumuloCluster accumulo, final Connector accumuloConn, final String ryaTablePrefix, final RyaSailRepository ryaRepo, final RepositoryConnection ryaConn, final MiniFluo fluo, final FluoClient fluoClient) throws DemoExecutionException {
    log.setLevel(Level.INFO);
    // 1. Introduce some RDF Statements that we are going to start with and
    // pause so the presenter can introduce this information to the audience.
    final Set<RyaStatement> relevantHistoricStatements = Sets.newHashSet(new RyaStatement(eve, talksTo, charlie), new RyaStatement(david, talksTo, alice), new RyaStatement(alice, worksAt, coffeeShop), new RyaStatement(bob, worksAt, coffeeShop));
    log.info("We add some Statements that are relevant to the query we will compute:");
    prettyLogStatements(relevantHistoricStatements);
    waitForEnter();
    log.info("We also some more Satements that aren't realted to the query we will compute");
    final Set<RyaStatement> otherHistoricStatements = Sets.newHashSet(new RyaStatement(henry, worksAt, burgerShop), new RyaStatement(irene, worksAt, burgerShop), new RyaStatement(justin, worksAt, burgerShop), new RyaStatement(kristi, worksAt, burgerShop), new RyaStatement(luke, worksAt, burgerShop), new RyaStatement(manny, worksAt, cupcakeShop), new RyaStatement(nate, worksAt, cupcakeShop), new RyaStatement(olivia, worksAt, cupcakeShop), new RyaStatement(paul, worksAt, cupcakeShop), new RyaStatement(ross, worksAt, cupcakeShop), new RyaStatement(henry, talksTo, irene), new RyaStatement(henry, talksTo, justin), new RyaStatement(kristi, talksTo, irene), new RyaStatement(luke, talksTo, irene), new RyaStatement(sally, talksTo, paul), new RyaStatement(sally, talksTo, ross), new RyaStatement(sally, talksTo, kristi), new RyaStatement(tim, talksTo, nate), new RyaStatement(tim, talksTo, paul), new RyaStatement(tim, talksTo, kristi));
    log.info("Theese statements will also be inserted into the core Rya tables:");
    prettyLogStatements(otherHistoricStatements);
    waitForEnter();
    // 2. Load the statements into the core Rya tables.
    log.info("Loading the historic statements into Rya...");
    loadDataIntoRya(ryaConn, relevantHistoricStatements);
    loadDataIntoRya(ryaConn, otherHistoricStatements);
    log.info("");
    // 3. Introduce the query that we're going to load into Fluo and pause so that the
    // presenter may show what they believe the expected output should be.
    final String sparql = "SELECT ?patron ?employee " + "WHERE { " + "?patron <http://talksTo> ?employee. " + "?employee <http://worksAt> <http://CoffeeShop>. " + "}";
    log.info("The following SPARQL query will be loaded into the Fluo application for incremental updates:");
    prettyLogSparql(sparql);
    waitForEnter();
    // 4. Write the query to Fluo and import the historic matches. Wait for the app to finish exporting results.
    log.info("Telling Fluo to maintain the query and import the historic Statement Pattern matches.");
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, ryaTablePrefix);
    final String pcjId;
    try {
        // Create the PCJ Index in Rya.
        pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo app to maintain it.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, ryaTablePrefix);
    } catch (MalformedQueryException | PcjException | RyaDAOException | UnsupportedQueryException e) {
        throw new DemoExecutionException("Error while using Fluo to compute and export historic matches, so the demo can not continue. Exiting.", e);
    }
    log.info("Waiting for the fluo application to finish exporting the initial results...");
    fluo.waitForObservers();
    log.info("Historic result exporting finished.");
    log.info("");
    // 5. Show that the Fluo app exported the results to the PCJ table in Accumulo.
    log.info("The following Binding Sets were exported to the PCJ with ID '" + pcjId + "' in Rya:");
    try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
        while (resultsIt.hasNext()) {
            log.info("    " + resultsIt.next());
        }
    } catch (final Exception e) {
        throw new DemoExecutionException("Could not fetch the PCJ's reuslts from Accumulo. Exiting.", e);
    }
    waitForEnter();
    // 6. Introduce some new Statements that we will stream into the Fluo app.
    final RyaStatement newLeft = new RyaStatement(george, talksTo, frank);
    final RyaStatement newRight = new RyaStatement(frank, worksAt, coffeeShop);
    final RyaStatement joinLeft = new RyaStatement(eve, talksTo, bob);
    final RyaStatement joinRight = new RyaStatement(charlie, worksAt, coffeeShop);
    final Set<RyaStatement> relevantstreamedStatements = Sets.newHashSet(newLeft, newRight, joinLeft, joinRight);
    log.info("We stream these relevant Statements into Fluo and the core Rya tables:");
    log.info(prettyFormat(newLeft) + "          - Part of a new result");
    log.info(prettyFormat(newRight) + "      - Other part of a new result");
    log.info(prettyFormat(joinLeft) + "               - Joins with a historic <http://talksTo> statement");
    log.info(prettyFormat(joinRight) + "    - Joins with a historic <http://worksA>t statement");
    waitForEnter();
    final Set<RyaStatement> otherStreamedStatements = Sets.newHashSet(new RyaStatement(alice, talksTo, tim), new RyaStatement(bob, talksTo, tim), new RyaStatement(charlie, talksTo, tim), new RyaStatement(frank, talksTo, tim), new RyaStatement(david, talksTo, tim), new RyaStatement(eve, talksTo, sally), new RyaStatement(george, talksTo, sally), new RyaStatement(henry, talksTo, sally), new RyaStatement(irene, talksTo, sally), new RyaStatement(justin, talksTo, sally), new RyaStatement(kristi, talksTo, manny), new RyaStatement(luke, talksTo, manny), new RyaStatement(manny, talksTo, paul), new RyaStatement(nate, talksTo, manny), new RyaStatement(olivia, talksTo, manny), new RyaStatement(paul, talksTo, kristi), new RyaStatement(ross, talksTo, kristi), new RyaStatement(sally, talksTo, kristi), new RyaStatement(olivia, talksTo, kristi), new RyaStatement(olivia, talksTo, kristi));
    log.info("We also stream these irrelevant Statements into Fluo and the core Rya tables:");
    prettyLogStatements(otherStreamedStatements);
    waitForEnter();
    // 7. Insert the new triples into the core Rya tables and the Fluo app.
    loadDataIntoRya(ryaConn, relevantstreamedStatements);
    loadDataIntoFluo(fluoClient, relevantstreamedStatements);
    log.info("Waiting for the fluo application to finish exporting the newly streamed results...");
    fluo.waitForObservers();
    log.info("Streamed result exporting finished.");
    log.info("");
    // 8. Show the new results have been exported to the PCJ table in Accumulo.
    log.info("The following Binding Sets were exported to the PCJ with ID '" + pcjId + "' in Rya:");
    try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
        while (resultsIt.hasNext()) {
            log.info("    " + resultsIt.next());
        }
    } catch (final Exception e) {
        throw new DemoExecutionException("Could not fetch the PCJ's reuslts from Accumulo. Exiting.", e);
    }
    log.info("");
}

Example 27 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class BatchIT method simpleJoinDelete.

@Test
public void simpleJoinDelete() throws Exception {
    final String sparql = "SELECT ?subject ?object1 ?object2 WHERE { ?subject <urn:predicate_1> ?object1; " + " <urn:predicate_2> ?object2 } ";
    try (FluoClient fluoClient = new FluoClientImpl(getFluoConfiguration())) {
        RyaURI subj = new RyaURI("urn:subject_1");
        RyaStatement statement1 = new RyaStatement(subj, new RyaURI("urn:predicate_1"), null);
        RyaStatement statement2 = new RyaStatement(subj, new RyaURI("urn:predicate_2"), null);
        Set<RyaStatement> statements1 = getRyaStatements(statement1, 5);
        Set<RyaStatement> statements2 = getRyaStatements(statement2, 5);
        // Create the PCJ table.
        final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(getAccumuloConnector(), getRyaInstanceName());
        final String pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo app to maintain the PCJ.
        String queryId = new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName()).getQueryId();
        List<String> ids = getNodeIdStrings(fluoClient, queryId);
        String joinId = ids.get(2);
        String rightSp = ids.get(4);
        QueryBindingSet bs = new QueryBindingSet();
        bs.addBinding("subject", vf.createURI("urn:subject_1"));
        bs.addBinding("object1", vf.createURI("urn:object_0"));
        VisibilityBindingSet vBs = new VisibilityBindingSet(bs);
        // create sharded span for deletion
        URI uri = vf.createURI("urn:subject_1");
        Bytes prefixBytes = BindingHashShardingFunction.getShardedScanPrefix(rightSp, uri);
        Span span = Span.prefix(prefixBytes);
        // Stream the data into Fluo.
        InsertTriples inserter = new InsertTriples();
        inserter.insert(fluoClient, statements1, Optional.absent());
        inserter.insert(fluoClient, statements2, Optional.absent());
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(25, 25, 25, 5, 5));
        JoinBatchInformation batch = JoinBatchInformation.builder().setBatchSize(1).setColumn(FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET).setSpan(span).setTask(Task.Delete).setJoinType(JoinType.NATURAL_JOIN).setSide(Side.LEFT).setBs(vBs).build();
        // Verify the end results of the query match the expected results.
        createSpanBatch(fluoClient, joinId, batch);
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(25, 25, 20, 5, 5));
    }
}

Example 28 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class GetPcjMetadataIT method getMetadataByQueryId.

@Test
public void getMetadataByQueryId() throws RepositoryException, MalformedQueryException, SailException, QueryEvaluationException, PcjException, NotInFluoException, NotInAccumuloException, RyaDAOException, UnsupportedQueryException {
    final String sparql = "SELECT ?x " + "WHERE { " + "?x <http://talksTo> <http://Eve>. " + "?x <http://worksAt> <http://Chipotle>." + "}";
    // Create the PCJ table.
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = pcjStorage.createPcj(sparql);
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Tell the Fluo app to maintain the PCJ.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Fetch the PCJ's Metadata through the GetPcjMetadata interactor.
        final String queryId = new ListQueryIds().listQueryIds(fluoClient).get(0);
        final PcjMetadata metadata = new GetPcjMetadata().getMetadata(pcjStorage, fluoClient, queryId);
        // Ensure the command returns the correct metadata.
        final Set<VariableOrder> varOrders = new ShiftVarOrderFactory().makeVarOrders(sparql);
        final PcjMetadata expected = new PcjMetadata(sparql, 0L, varOrders);
        assertEquals(expected, metadata);
    }
}

Example 29 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class GetPcjMetadataIT method getAllMetadata.

@Test
public void getAllMetadata() throws MalformedQueryException, SailException, QueryEvaluationException, PcjException, NotInFluoException, NotInAccumuloException, AccumuloException, AccumuloSecurityException, RyaDAOException, UnsupportedQueryException {
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Add a couple of queries to Accumulo.
        final String q1Sparql = "SELECT ?x " + "WHERE { " + "?x <http://talksTo> <http://Eve>. " + "?x <http://worksAt> <http://Chipotle>." + "}";
        final String q1PcjId = pcjStorage.createPcj(q1Sparql);
        final CreateFluoPcj createPcj = new CreateFluoPcj();
        createPcj.withRyaIntegration(q1PcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        final String q2Sparql = "SELECT ?x ?y " + "WHERE { " + "?x <http://talksTo> ?y. " + "?y <http://worksAt> <http://Chipotle>." + "}";
        final String q2PcjId = pcjStorage.createPcj(q2Sparql);
        createPcj.withRyaIntegration(q2PcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Ensure the command returns the correct metadata.
        final Set<PcjMetadata> expected = new HashSet<>();
        final Set<VariableOrder> q1VarOrders = new ShiftVarOrderFactory().makeVarOrders(q1Sparql);
        final Set<VariableOrder> q2VarOrders = new ShiftVarOrderFactory().makeVarOrders(q2Sparql);
        expected.add(new PcjMetadata(q1Sparql, 0L, q1VarOrders));
        expected.add(new PcjMetadata(q2Sparql, 0L, q2VarOrders));
        final Map<String, PcjMetadata> metadata = new GetPcjMetadata().getMetadata(pcjStorage, fluoClient);
        assertEquals(expected, Sets.newHashSet(metadata.values()));
    }
}

Example 30 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class QueryBenchmarkRunIT method createTestPCJ.

private static void createTestPCJ(final RyaClient ryaClient) throws Exception {
    // Create an empty PCJ within the Rya instance's PCJ storage for the test query.
    try (final PrecomputedJoinStorage pcjs = new AccumuloPcjStorage(cluster.getConnector(ACCUMULO_USER, ACCUMULO_PASSWORD), RYA_INSTANCE_NAME)) {
        final String pcjId = pcjs.createPcj(SPARQL_QUERY);
        // Batch update the PCJ using the Rya Client.
        ryaClient.getBatchUpdatePCJ().batchUpdate(RYA_INSTANCE_NAME, pcjId);
    }
}