Examples with CreateFluoPcj org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj used on opensource projects

Example 16 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class RyaInputIncrementalUpdateIT method streamResultsThroughRya.

/**
 * Ensure historic matches are included in the result.
 */
@Test
public void streamResultsThroughRya() throws Exception {
    // A query that finds people who talk to Eve and work at Chipotle.
    final String sparql = "SELECT ?x " + "WHERE { " + "?x <http://talksTo> <http://Eve>. " + "?x <http://worksAt> <http://Chipotle>." + "}";
    // Triples that are loaded into Rya before the PCJ is created.
    final ValueFactory vf = new ValueFactoryImpl();
    final Set<Statement> historicTriples = Sets.newHashSet(vf.createStatement(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")), vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")), vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://talksTo"), vf.createURI("http://Eve")), vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://helps"), vf.createURI("http://Kevin")), vf.createStatement(vf.createURI("http://Bob"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")), vf.createStatement(vf.createURI("http://Charlie"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")), vf.createStatement(vf.createURI("http://Eve"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")), vf.createStatement(vf.createURI("http://David"), vf.createURI("http://worksAt"), vf.createURI("http://Chipotle")));
    // The expected results of the SPARQL query once the PCJ has been
    // computed.
    final Set<BindingSet> expected = new HashSet<>();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("x", vf.createURI("http://Bob"));
    expected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("x", vf.createURI("http://Charlie"));
    expected.add(bs);
    // Create the PCJ table.
    final Connector accumuloConn = super.getAccumuloConnector();
    final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = pcjStorage.createPcj(sparql);
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Tell the Fluo app to maintain the PCJ.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Verify the end results of the query match the expected results.
        super.getMiniFluo().waitForObservers();
        // Load the historic data into Rya.
        final SailRepositoryConnection ryaConn = super.getRyaSailRepository().getConnection();
        for (final Statement triple : historicTriples) {
            ryaConn.add(triple);
        }
        super.getMiniFluo().waitForObservers();
        final Set<BindingSet> results = new HashSet<>();
        try (CloseableIterator<BindingSet> resultIt = pcjStorage.listResults(pcjId)) {
            while (resultIt.hasNext()) {
                results.add(resultIt.next());
            }
        }
        assertEquals(expected, results);
    }
}

Example 17 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class StreamingTestIT method testRandomStreamingIngest.

@Test
public void testRandomStreamingIngest() throws Exception {
    final String sparql = "select ?name ?uuid where { " + "?uuid <http://pred1> ?name ; " + "<http://pred2> \"literal\"." + "}";
    try (FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Create the PCJ table.
        final Connector accumuloConn = super.getAccumuloConnector();
        final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
        final String pcjId = pcjStorage.createPcj(sparql);
        // Task the Fluo app with the PCJ.
        new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Add Statements to the Fluo app.
        log.info("Adding Join Pairs...");
        addRandomQueryStatementPairs(100);
        super.getMiniFluo().waitForObservers();
        int resultCount = 0;
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            while (resultsIt.hasNext()) {
                resultCount++;
                resultsIt.next();
            }
        }
        // Show the correct number of Binding Sets were created for the PCJ.
        assertEquals(100, resultCount);
    }
}

Example 18 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class PcjVisibilityIT method createWithVisibilityFluo.

@Test
public void createWithVisibilityFluo() throws Exception {
    final String sparql = "SELECT ?customer ?worker ?city " + "{ " + "?customer <http://talksTo> ?worker. " + "?worker <http://livesIn> ?city. " + "?worker <http://worksAt> <http://Chipotle>. " + "}";
    // Triples that will be streamed into Fluo after the PCJ has been created.
    final Map<RyaStatement, String> streamedTriples = new HashMap<>();
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Bob")), "A&B");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "A");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Bob"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "B");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Charlie")), "B&C");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "B");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Charlie"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "C");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://David")), "C&D");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://David"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "C");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://David"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "D");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Alice"), new RyaURI("http://talksTo"), new RyaURI("http://Eve")), "D&E");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://livesIn"), new RyaURI("http://Leeds")), "D");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Eve"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "E");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://talksTo"), new RyaURI("http://Alice")), "");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://livesIn"), new RyaURI("http://London")), "");
    addStatementVisibilityEntry(streamedTriples, new RyaStatement(new RyaURI("http://Frank"), new RyaURI("http://worksAt"), new RyaURI("http://Chipotle")), "");
    final Connector accumuloConn = super.getAccumuloConnector();
    // Create the PCJ Table in Accumulo.
    final PrecomputedJoinStorage rootStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName());
    final String pcjId = rootStorage.createPcj(sparql);
    try (final FluoClient fluoClient = FluoFactory.newClient(super.getFluoConfiguration())) {
        // Create the PCJ in Fluo.
        new CreateFluoPcj().withRyaIntegration(pcjId, rootStorage, fluoClient, accumuloConn, getRyaInstanceName());
        // Stream the data into Fluo.
        for (final RyaStatement statement : streamedTriples.keySet()) {
            final Optional<String> visibility = Optional.of(streamedTriples.get(statement));
            new InsertTriples().insert(fluoClient, statement, visibility);
        }
    }
    // Fetch the exported results from Accumulo once the observers finish working.
    super.getMiniFluo().waitForObservers();
    setupTestUsers(accumuloConn, getRyaInstanceName(), pcjId);
    // Verify ABCDE using root.
    final Set<BindingSet> rootResults = toSet(rootStorage.listResults(pcjId));
    final Set<BindingSet> rootExpected = Sets.newHashSet();
    MapBindingSet bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://Bob"));
    bs.addBinding("city", VF.createURI("http://London"));
    rootExpected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://Charlie"));
    bs.addBinding("city", VF.createURI("http://London"));
    rootExpected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://Eve"));
    bs.addBinding("city", VF.createURI("http://Leeds"));
    rootExpected.add(bs);
    bs = new MapBindingSet();
    bs.addBinding("customer", VF.createURI("http://Alice"));
    bs.addBinding("worker", VF.createURI("http://David"));
    bs.addBinding("city", VF.createURI("http://London"));
    rootExpected.add(bs);
    assertEquals(rootExpected, rootResults);
    final MiniAccumuloCluster cluster = super.getMiniAccumuloCluster();
    // Verify AB
    final Connector abConn = cluster.getConnector("abUser", "password");
    try (final PrecomputedJoinStorage abStorage = new AccumuloPcjStorage(abConn, getRyaInstanceName())) {
        final Set<BindingSet> abResults = toSet(abStorage.listResults(pcjId));
        final Set<BindingSet> abExpected = Sets.newHashSet();
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Bob"));
        bs.addBinding("city", VF.createURI("http://London"));
        abExpected.add(bs);
        assertEquals(abExpected, abResults);
    }
    // Verify ABC
    final Connector abcConn = cluster.getConnector("abcUser", "password");
    try (final PrecomputedJoinStorage abcStorage = new AccumuloPcjStorage(abcConn, getRyaInstanceName())) {
        final Set<BindingSet> abcResults = toSet(abcStorage.listResults(pcjId));
        final Set<BindingSet> abcExpected = Sets.newHashSet();
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Bob"));
        bs.addBinding("city", VF.createURI("http://London"));
        abcExpected.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Charlie"));
        bs.addBinding("city", VF.createURI("http://London"));
        abcExpected.add(bs);
        assertEquals(abcExpected, abcResults);
    }
    // Verify ADE
    final Connector adeConn = cluster.getConnector("adeUser", "password");
    try (final PrecomputedJoinStorage adeStorage = new AccumuloPcjStorage(adeConn, getRyaInstanceName())) {
        final Set<BindingSet> adeResults = toSet(adeStorage.listResults(pcjId));
        final Set<BindingSet> adeExpected = Sets.newHashSet();
        bs = new MapBindingSet();
        bs.addBinding("customer", VF.createURI("http://Alice"));
        bs.addBinding("worker", VF.createURI("http://Eve"));
        bs.addBinding("city", VF.createURI("http://Leeds"));
        adeExpected.add(bs);
        assertEquals(adeExpected, adeResults);
    }
    // Verify no auths.
    final Connector noAuthConn = cluster.getConnector("noAuth", "password");
    try (final PrecomputedJoinStorage noAuthStorage = new AccumuloPcjStorage(noAuthConn, getRyaInstanceName())) {
        final Set<BindingSet> noAuthResults = toSet(noAuthStorage.listResults(pcjId));
        assertTrue(noAuthResults.isEmpty());
    }
}

Example 19 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class NewQueryCommand method execute.

@Override
public void execute(final Connector accumulo, final String ryaTablePrefix, final RyaSailRepository rya, final FluoClient fluo, final String[] args) throws ArgumentsException, ExecutionException, UnsupportedQueryException {
    checkNotNull(accumulo);
    checkNotNull(fluo);
    checkNotNull(args);
    log.trace("Executing the New Query Command...");
    // Parse the command line arguments.
    final Parameters params = new Parameters();
    try {
        new JCommander(params, args);
    } catch (final ParameterException e) {
        throw new ArgumentsException("Could not create a new query because of invalid command line parameters.", e);
    }
    // Load the request from the file into memory.
    log.trace("Loading the query found in file '" + params.queryRequestFile + "' into the client app.");
    ParsedQueryRequest request = null;
    try {
        final Path requestFile = Paths.get(params.queryRequestFile);
        final String requestText = IOUtils.toString(Files.newInputStream(requestFile));
        request = ParsedQueryRequest.parse(requestText);
    } catch (final IOException e) {
        throw new ExecutionException("Could not load the query request into memory.", e);
    }
    // Load the query into the Fluo app.
    log.trace("SPARQL Query: " + request.getQuery());
    log.trace("Var Orders: " + request.getVarOrders());
    log.trace("Loading these values into the Fluo app.");
    final CreateFluoPcj createPcj = new CreateFluoPcj();
    try {
        // Create the PCJ in Rya.
        final String sparql = request.getQuery();
        final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumulo, ryaTablePrefix);
        final String pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo PCJ Updater app to maintain the PCJ.
        createPcj.withRyaIntegration(pcjId, pcjStorage, fluo, accumulo, ryaTablePrefix);
    } catch (MalformedQueryException | PcjException | RyaDAOException e) {
        throw new ExecutionException("Could not create and load historic matches into the the Fluo app for the query.", e);
    }
    log.trace("Finished executing the New Query Command.");
}

Example 20 with CreateFluoPcj

use of org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj in project incubator-rya by apache.

the class BatchIT method simpleScanDelete.

@Test
public void simpleScanDelete() throws Exception {
    final String sparql = "SELECT ?subject ?object1 ?object2 WHERE { ?subject <urn:predicate_1> ?object1; " + " <urn:predicate_2> ?object2 } ";
    try (FluoClient fluoClient = new FluoClientImpl(getFluoConfiguration())) {
        RyaURI subj = new RyaURI("urn:subject_1");
        RyaStatement statement1 = new RyaStatement(subj, new RyaURI("urn:predicate_1"), null);
        RyaStatement statement2 = new RyaStatement(subj, new RyaURI("urn:predicate_2"), null);
        Set<RyaStatement> statements1 = getRyaStatements(statement1, 10);
        Set<RyaStatement> statements2 = getRyaStatements(statement2, 10);
        // Create the PCJ table.
        final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(getAccumuloConnector(), getRyaInstanceName());
        final String pcjId = pcjStorage.createPcj(sparql);
        // Tell the Fluo app to maintain the PCJ.
        String queryId = new CreateFluoPcj().withRyaIntegration(pcjId, pcjStorage, fluoClient, getAccumuloConnector(), getRyaInstanceName()).getQueryId();
        List<String> ids = getNodeIdStrings(fluoClient, queryId);
        List<String> prefixes = Arrays.asList("urn:subject_1", "urn:subject_1", "urn:object", "urn:subject_1", "urn:subject_1");
        // Stream the data into Fluo.
        InsertTriples inserter = new InsertTriples();
        inserter.insert(fluoClient, statements1, Optional.absent());
        inserter.insert(fluoClient, statements2, Optional.absent());
        // Verify the end results of the query match the expected results.
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(100, 100, 100, 10, 10));
        createSpanBatches(fluoClient, ids, prefixes, 10);
        getMiniFluo().waitForObservers();
        verifyCounts(fluoClient, ids, Arrays.asList(0, 0, 0, 0, 0));
    }
}