Examples with AccumuloPcjStorage org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage used on opensource projects

Example 21 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class ListQueriesCommand method execute.

@Override
public void execute(final Connector accumulo, final String ryaTablePrefix, final RyaSailRepository rya, final FluoClient fluo, final String[] args) throws ArgumentsException, ExecutionException {
    checkNotNull(accumulo);
    checkNotNull(fluo);
    checkNotNull(args);
    log.trace("Executing the List Queries Command...");
    // Parse the command line arguments.
    final Parameters params = new Parameters();
    try {
        new JCommander(params, args);
    } catch (final ParameterException e) {
        throw new ArgumentsException("Could not list the queries because of invalid command line parameters.", e);
    }
    // Fetch the PCJ metadata that will be included in the report.
    final GetPcjMetadata getPcjMetadata = new GetPcjMetadata();
    final Map<String, PcjMetadata> metadata = new HashMap<String, PcjMetadata>();
    try {
        final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumulo, ryaTablePrefix);
        if (params.queryId != null) {
            log.trace("Fetch the PCJ Metadata from Accumulo for Query ID '" + params.queryId + "'.");
            metadata.put(params.queryId, getPcjMetadata.getMetadata(pcjStorage, fluo, params.queryId));
        } else {
            log.trace("Fetch the PCJ Metadata from Accumulo for all queries that are being updated by Fluo.");
            metadata.putAll(getPcjMetadata.getMetadata(pcjStorage, fluo));
        }
    } catch (NotInFluoException | NotInAccumuloException e) {
        throw new ExecutionException("Could not fetch some of the metadata required to build the report.", e);
    }
    // Write the metadata to the console.
    log.trace("Rendering the queries report...");
    if (metadata.isEmpty()) {
        System.out.println("No queries are being tracked by Fluo.");
    } else {
        final PcjMetadataRenderer renderer = new PcjMetadataRenderer();
        try {
            final String report = renderer.render(metadata);
            System.out.println("The number of Queries that are being tracked by Fluo: " + metadata.size());
            System.out.println(report);
        } catch (final Exception e) {
            throw new ExecutionException("Unable to render the query metadata report for output.", e);
        }
    }
    log.trace("Finished executing the List Queries Command.");
}

Example 22 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class FluoAndHistoricPcjsDemo method execute.

@Override
public void execute(final MiniAccumuloCluster accumulo, final Connector accumuloConn, final String ryaTablePrefix, final RyaSailRepository ryaRepo, final RepositoryConnection ryaConn, final MiniFluo fluo, final FluoClient fluoClient) throws DemoExecutionException {
    log.setLevel(Level.INFO);
    // 1. Introduce some RDF Statements that we are going to start with and
    // pause so the presenter can introduce this information to the audience.
    final Set<RyaStatement> relevantHistoricStatements = Sets.newHashSet(new RyaStatement(eve, talksTo, charlie), new RyaStatement(david, talksTo, alice), new RyaStatement(alice, worksAt, coffeeShop), new RyaStatement(bob, worksAt, coffeeShop));
    log.info("We add some Statements that are relevant to the query we will compute:");
    prettyLogStatements(relevantHistoricStatements);
    waitForEnter();
    log.info("We also some more Satements that aren't realted to the query we will compute");
    final Set<RyaStatement> otherHistoricStatements = Sets.newHashSet(new RyaStatement(henry, worksAt, burgerShop), new RyaStatement(irene, worksAt, burgerShop), new RyaStatement(justin, worksAt, burgerShop), new RyaStatement(kristi, worksAt, burgerShop), new RyaStatement(luke, worksAt, burgerShop), new RyaStatement(manny, worksAt, cupcakeShop), new RyaStatement(nate, worksAt, cupcakeShop), new RyaStatement(olivia, worksAt, cupcakeShop), new RyaStatement(paul, worksAt, cupcakeShop), new RyaStatement(ross, worksAt, cupcakeShop), new RyaStatement(henry, talksTo, irene), new RyaStatement(henry, talksTo, justin), new RyaStatement(kristi, talksTo, irene), new RyaStatement(luke, talksTo, irene), new RyaStatement(sally, talksTo, paul), new RyaStatement(sally, talksTo, ross), new RyaStatement(sally, talksTo, kristi), new RyaStatement(tim, talksTo, nate), new RyaStatement(tim, talksTo, paul), new RyaStatement(tim, talksTo, kristi));
    log.info("Theese statements will also be inserted into the core Rya tables:");
    prettyLogStatements(otherHistoricStatements);
    waitForEnter();
    // 2. Load the statements into the core Rya tables.
    log.info("Loading the historic statements into Rya...");
    loadDataIntoRya(ryaConn, relevantHistoricStatements);
    loadDataIntoRya(ryaConn, otherHistoricStatements);
    log.info("");
    // 3. Introduce the query that we're going to load into Fluo and pause so that the
    // presenter may show what they believe the expected output should be.
    final String sparql = "SELECT ?patron ?employee " + "WHERE { " + "?patron <http://talksTo> ?employee. " + "?employee <http://worksAt> <http://CoffeeShop>. " + "}";
    log.info("The following SPARQL query will be loaded into the Fluo application for incremental updates:");
    prettyLogSparql(sparql);
    waitForEnter();
    // 4. Write the query to Fluo and import the historic matches. Wait for the app to finish exporting results.
    log.info("Telling Fluo to maintain the query and import the historic Statement Pattern matches.");
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, ryaTablePrefix);
    final String pcjId;
    try {
        // Create the PCJ Index in Rya.
        pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo app to maintain it.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, ryaTablePrefix);
    } catch (MalformedQueryException | PcjException | RyaDAOException | UnsupportedQueryException e) {
        throw new DemoExecutionException("Error while using Fluo to compute and export historic matches, so the demo can not continue. Exiting.", e);
    }
    log.info("Waiting for the fluo application to finish exporting the initial results...");
    fluo.waitForObservers();
    log.info("Historic result exporting finished.");
    log.info("");
    // 5. Show that the Fluo app exported the results to the PCJ table in Accumulo.
    log.info("The following Binding Sets were exported to the PCJ with ID '" + pcjId + "' in Rya:");
    try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
        while (resultsIt.hasNext()) {
            log.info("    " + resultsIt.next());
        }
    } catch (final Exception e) {
        throw new DemoExecutionException("Could not fetch the PCJ's reuslts from Accumulo. Exiting.", e);
    }
    waitForEnter();
    // 6. Introduce some new Statements that we will stream into the Fluo app.
    final RyaStatement newLeft = new RyaStatement(george, talksTo, frank);
    final RyaStatement newRight = new RyaStatement(frank, worksAt, coffeeShop);
    final RyaStatement joinLeft = new RyaStatement(eve, talksTo, bob);
    final RyaStatement joinRight = new RyaStatement(charlie, worksAt, coffeeShop);
    final Set<RyaStatement> relevantstreamedStatements = Sets.newHashSet(newLeft, newRight, joinLeft, joinRight);
    log.info("We stream these relevant Statements into Fluo and the core Rya tables:");
    log.info(prettyFormat(newLeft) + "          - Part of a new result");
    log.info(prettyFormat(newRight) + "      - Other part of a new result");
    log.info(prettyFormat(joinLeft) + "               - Joins with a historic <http://talksTo> statement");
    log.info(prettyFormat(joinRight) + "    - Joins with a historic <http://worksA>t statement");
    waitForEnter();
    final Set<RyaStatement> otherStreamedStatements = Sets.newHashSet(new RyaStatement(alice, talksTo, tim), new RyaStatement(bob, talksTo, tim), new RyaStatement(charlie, talksTo, tim), new RyaStatement(frank, talksTo, tim), new RyaStatement(david, talksTo, tim), new RyaStatement(eve, talksTo, sally), new RyaStatement(george, talksTo, sally), new RyaStatement(henry, talksTo, sally), new RyaStatement(irene, talksTo, sally), new RyaStatement(justin, talksTo, sally), new RyaStatement(kristi, talksTo, manny), new RyaStatement(luke, talksTo, manny), new RyaStatement(manny, talksTo, paul), new RyaStatement(nate, talksTo, manny), new RyaStatement(olivia, talksTo, manny), new RyaStatement(paul, talksTo, kristi), new RyaStatement(ross, talksTo, kristi), new RyaStatement(sally, talksTo, kristi), new RyaStatement(olivia, talksTo, kristi), new RyaStatement(olivia, talksTo, kristi));
    log.info("We also stream these irrelevant Statements into Fluo and the core Rya tables:");
    prettyLogStatements(otherStreamedStatements);
    waitForEnter();
    // 7. Insert the new triples into the core Rya tables and the Fluo app.
    loadDataIntoRya(ryaConn, relevantstreamedStatements);
    loadDataIntoFluo(fluoClient, relevantstreamedStatements);
    log.info("Waiting for the fluo application to finish exporting the newly streamed results...");
    fluo.waitForObservers();
    log.info("Streamed result exporting finished.");
    log.info("");
    // 8. Show the new results have been exported to the PCJ table in Accumulo.
    log.info("The following Binding Sets were exported to the PCJ with ID '" + pcjId + "' in Rya:");
    try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
        while (resultsIt.hasNext()) {
            log.info("    " + resultsIt.next());
        }
    } catch (final Exception e) {
        throw new DemoExecutionException("Could not fetch the PCJ's reuslts from Accumulo. Exiting.", e);
    }
    log.info("");
}

Example 23 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class BatchIT method simpleJoinDelete.

@Test
public void simpleJoinDelete() throws Exception {
    final String sparql = "SELECT ?subject ?object1 ?object2 WHERE { ?subject <urn:predicate_1> ?object1; " + " <urn:predicate_2> ?object2 } ";
    try (FluoClient fluoClient = new FluoClientImpl(getFluoConfiguration())) {
        RyaURI subj = new RyaURI("urn:subject_1");
        RyaStatement statement1 = new RyaStatement(subj, new RyaURI("urn:predicate_1"), null);
        RyaStatement statement2 = new RyaStatement(subj, new RyaURI("urn:predicate_2"), null);
        Set<RyaStatement> statements1 = getRyaStatements(statement1, 5);
        Set<RyaStatement> statements2 = getRyaStatements(statement2, 5);
        // Create the PCJ table.
        final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(getAccumuloConnector(), getRyaInstanceName());
        final String pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo app to maintain the PCJ.
        String queryId = new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName()).getQueryId();
        List<String> ids = getNodeIdStrings(fluoClient, queryId);
        String joinId = ids.get(2);
        String rightSp = ids.get(4);
        QueryBindingSet bs = new QueryBindingSet();
        bs.addBinding("subject", vf.createURI("urn:subject_1"));
        bs.addBinding("object1", vf.createURI("urn:object_0"));
        VisibilityBindingSet vBs = new VisibilityBindingSet(bs);
        // create sharded span for deletion
        URI uri = vf.createURI("urn:subject_1");
        Bytes prefixBytes = BindingHashShardingFunction.getShardedScanPrefix(rightSp, uri);
        Span span = Span.prefix(prefixBytes);
        // Stream the data into Fluo.
        InsertTriples inserter = new InsertTriples();
        inserter.insert(fluoClient, statements1, Optional.absent());
        inserter.insert(fluoClient, statements2, Optional.absent());
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(25, 25, 25, 5, 5));
        JoinBatchInformation batch = JoinBatchInformation.builder().setBatchSize(1).setColumn(FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET).setSpan(span).setTask(Task.Delete).setJoinType(JoinType.NATURAL_JOIN).setSide(Side.LEFT).setBs(vBs).build();
        // Verify the end results of the query match the expected results.
        createSpanBatch(fluoClient, joinId, batch);
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(25, 25, 20, 5, 5));
    }
}

Example 24 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class GetPcjMetadataIT method getMetadataByQueryId.

@Test
public void getMetadataByQueryId() throws RepositoryException, MalformedQueryException, SailException, QueryEvaluationException, PcjException, NotInFluoException, NotInAccumuloException, RyaDAOException, UnsupportedQueryException {
    final String sparql = "SELECT ?x " + "WHERE { " + "?x <http://talksTo> <http://Eve>. " + "?x <http://worksAt> <http://Chipotle>." + "}";
    // Create the PCJ table.
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = pcjStorage.createPcj(sparql);
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Tell the Fluo app to maintain the PCJ.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Fetch the PCJ's Metadata through the GetPcjMetadata interactor.
        final String queryId = new ListQueryIds().listQueryIds(fluoClient).get(0);
        final PcjMetadata metadata = new GetPcjMetadata().getMetadata(pcjStorage, fluoClient, queryId);
        // Ensure the command returns the correct metadata.
        final Set<VariableOrder> varOrders = new ShiftVarOrderFactory().makeVarOrders(sparql);
        final PcjMetadata expected = new PcjMetadata(sparql, 0L, varOrders);
        assertEquals(expected, metadata);
    }
}

Example 25 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class GetPcjMetadataIT method getAllMetadata.

@Test
public void getAllMetadata() throws MalformedQueryException, SailException, QueryEvaluationException, PcjException, NotInFluoException, NotInAccumuloException, AccumuloException, AccumuloSecurityException, RyaDAOException, UnsupportedQueryException {
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Add a couple of queries to Accumulo.
        final String q1Sparql = "SELECT ?x " + "WHERE { " + "?x <http://talksTo> <http://Eve>. " + "?x <http://worksAt> <http://Chipotle>." + "}";
        final String q1PcjId = pcjStorage.createPcj(q1Sparql);
        final CreateFluoPcj createPcj = new CreateFluoPcj();
        createPcj.withRyaIntegration(q1PcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        final String q2Sparql = "SELECT ?x ?y " + "WHERE { " + "?x <http://talksTo> ?y. " + "?y <http://worksAt> <http://Chipotle>." + "}";
        final String q2PcjId = pcjStorage.createPcj(q2Sparql);
        createPcj.withRyaIntegration(q2PcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Ensure the command returns the correct metadata.
        final Set<PcjMetadata> expected = new HashSet<>();
        final Set<VariableOrder> q1VarOrders = new ShiftVarOrderFactory().makeVarOrders(q1Sparql);
        final Set<VariableOrder> q2VarOrders = new ShiftVarOrderFactory().makeVarOrders(q2Sparql);
        expected.add(new PcjMetadata(q1Sparql, 0L, q1VarOrders));
        expected.add(new PcjMetadata(q2Sparql, 0L, q2VarOrders));
        final Map<String, PcjMetadata> metadata = new GetPcjMetadata().getMetadata(pcjStorage, fluoClient);
        assertEquals(expected, Sets.newHashSet(metadata.values()));
    }
}