Examples with MongoPcjStorage org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage used on opensource projects

Example 1 with MongoPcjStorage

use of org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage in project incubator-rya by apache.

the class MongoDeletePCJ method deletePCJ.

@Override
public void deletePCJ(final String ryaInstanceName, final String pcjId) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(ryaInstanceName);
    requireNonNull(pcjId);
    // Ensure the Rya Instance exists.
    if (!instanceExists.exists(ryaInstanceName)) {
        throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", ryaInstanceName));
    }
    try (final MongoPcjStorage pcjStore = new MongoPcjStorage(mongoClient, ryaInstanceName)) {
        pcjStore.dropPcj(pcjId);
    } catch (final PCJStorageException e) {
        throw new RyaClientException("Unable to drop PCJ : " + pcjId, e);
    }
}

Example 2 with MongoPcjStorage

use of org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage in project incubator-rya by apache.

the class MongoBatchUpdatePCJIT method batchUpdate.

@Test
public void batchUpdate() throws Exception {
    // Setup a Rya Client.
    final MongoConnectionDetails connectionDetails = getConnectionDetails();
    final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient());
    // Install an instance of Rya on the mini accumulo cluster.
    ryaClient.getInstall().install(conf.getRyaInstanceName(), InstallConfiguration.builder().setEnablePcjIndex(true).build());
    // Load some statements into the Rya instance.
    final ValueFactory vf = ValueFactoryImpl.getInstance();
    final Collection<Statement> statements = new ArrayList<>();
    statements.add(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:likes"), vf.createURI("urn:icecream")));
    statements.add(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:likes"), vf.createURI("urn:icecream")));
    statements.add(vf.createStatement(vf.createURI("urn:Charlie"), vf.createURI("urn:likes"), vf.createURI("urn:icecream")));
    statements.add(vf.createStatement(vf.createURI("urn:David"), vf.createURI("urn:likes"), vf.createURI("urn:icecream")));
    statements.add(vf.createStatement(vf.createURI("urn:Eve"), vf.createURI("urn:likes"), vf.createURI("urn:icecream")));
    statements.add(vf.createStatement(vf.createURI("urn:Frank"), vf.createURI("urn:likes"), vf.createURI("urn:icecream")));
    statements.add(vf.createStatement(vf.createURI("urn:George"), vf.createURI("urn:likes"), vf.createURI("urn:icecream")));
    statements.add(vf.createStatement(vf.createURI("urn:Hillary"), vf.createURI("urn:likes"), vf.createURI("urn:icecream")));
    statements.add(vf.createStatement(vf.createURI("urn:Alice"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue")));
    statements.add(vf.createStatement(vf.createURI("urn:Bob"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue")));
    statements.add(vf.createStatement(vf.createURI("urn:Charlie"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue")));
    statements.add(vf.createStatement(vf.createURI("urn:David"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue")));
    statements.add(vf.createStatement(vf.createURI("urn:Eve"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue")));
    statements.add(vf.createStatement(vf.createURI("urn:Frank"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue")));
    statements.add(vf.createStatement(vf.createURI("urn:George"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:green")));
    statements.add(vf.createStatement(vf.createURI("urn:Hillary"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:brown")));
    ryaClient.getLoadStatements().loadStatements(conf.getRyaInstanceName(), statements);
    try (final PrecomputedJoinStorage pcjStorage = new MongoPcjStorage(getMongoClient(), conf.getRyaInstanceName())) {
        // Create a PCJ for a SPARQL query.
        final String sparql = "SELECT ?name WHERE { ?name <urn:likes> <urn:icecream> . ?name <urn:hasEyeColor> <urn:blue> . }";
        final String pcjId = pcjStorage.createPcj(sparql);
        // Run the test.
        ryaClient.getBatchUpdatePCJ().batchUpdate(conf.getRyaInstanceName(), pcjId);
        // Verify the correct results were loaded into the PCJ table.
        final Set<BindingSet> expectedResults = new HashSet<>();
        MapBindingSet bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Alice"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Bob"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Charlie"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:David"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Eve"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Frank"));
        expectedResults.add(bs);
        final Set<BindingSet> results = new HashSet<>();
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            while (resultsIt.hasNext()) {
                results.add(resultsIt.next());
            }
        }
        assertEquals(expectedResults, results);
    }
}

Example 3 with MongoPcjStorage

use of org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage in project incubator-rya by apache.

the class MongoPcjIndexSetProvider method getIndices.

@Override
protected List<ExternalTupleSet> getIndices() throws PcjIndexSetException {
    try {
        final StatefulMongoDBRdfConfiguration mongoConf = (StatefulMongoDBRdfConfiguration) conf;
        final MongoClient client = mongoConf.getMongoClient();
        final MongoPcjDocuments pcjDocs = new MongoPcjDocuments(client, mongoConf.getRyaInstanceName());
        List<String> documents = null;
        documents = mongoConf.getPcjTables();
        // this maps associates pcj document name with pcj sparql query
        final Map<String, String> indexDocuments = Maps.newLinkedHashMap();
        try (final PrecomputedJoinStorage storage = new MongoPcjStorage(client, mongoConf.getRyaInstanceName())) {
            final boolean docsProvided = documents != null && !documents.isEmpty();
            if (docsProvided) {
                // if tables provided, associate table name with sparql
                for (final String doc : documents) {
                    indexDocuments.put(doc, storage.getPcjMetadata(doc).getSparql());
                }
            } else if (hasRyaDetails()) {
                // If this is a newer install of Rya, and it has PCJ Details, then
                // use those.
                final List<String> ids = storage.listPcjs();
                for (final String pcjId : ids) {
                    indexDocuments.put(pcjId, storage.getPcjMetadata(pcjId).getSparql());
                }
            } else {
                // Otherwise figure it out by getting document IDs.
                documents = pcjDocs.listPcjDocuments();
                for (final String pcjId : documents) {
                    if (pcjId.startsWith("INDEX")) {
                        indexDocuments.put(pcjId, pcjDocs.getPcjMetadata(pcjId).getSparql());
                    }
                }
            }
        }
        final List<ExternalTupleSet> index = Lists.newArrayList();
        if (indexDocuments.isEmpty()) {
            log.info("No Index found");
        } else {
            for (final String pcjID : indexDocuments.keySet()) {
                final String indexSparqlString = indexDocuments.get(pcjID);
                index.add(new MongoPcjQueryNode(indexSparqlString, pcjID, pcjDocs));
            }
        }
        return index;
    } catch (final PCJStorageException | MalformedQueryException e) {
        throw new PcjIndexSetException("Failed to get indicies for this PCJ index.", e);
    }
}

Example 4 with MongoPcjStorage

use of org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage in project incubator-rya by apache.

the class MongoBatchUpdatePCJ method updatePCJResults.

private void updatePCJResults(final String ryaInstanceName, final String pcjId, final MongoClient client) throws InstanceDoesNotExistException, PCJDoesNotExistException, RyaClientException {
    // Things that have to be closed before we exit.
    Sail sail = null;
    SailConnection sailConn = null;
    try (final PrecomputedJoinStorage pcjStorage = new MongoPcjStorage(client, ryaInstanceName)) {
        // Create an instance of Sail backed by the Rya instance.
        sail = connectToRya(ryaInstanceName);
        final SailRepository sailRepo = new SailRepository(sail);
        final SailRepositoryConnection sailRepoConn = sailRepo.getConnection();
        // Purge the old results from the PCJ.
        try {
            pcjStorage.purge(pcjId);
        } catch (final PCJStorageException e) {
            throw new RyaClientException("Could not batch update PCJ with ID '" + pcjId + "' because the old " + "results could not be purged from it.", e);
        }
        // Parse the PCJ's SPARQL query.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        final String sparql = metadata.getSparql();
        sailConn = sail.getConnection();
        final TupleQuery tupleQuery = sailRepoConn.prepareTupleQuery(QueryLanguage.SPARQL, sparql);
        // Execute the query.
        final List<VisibilityBindingSet> batch = new ArrayList<>(1000);
        tupleQuery.evaluate(new TupleQueryResultHandlerBase() {

            @Override
            public void handleSolution(final BindingSet bindingSet) throws TupleQueryResultHandlerException {
                final VisibilityBindingSet result = new VisibilityBindingSet(bindingSet, "");
                log.warn("Visibility information on the binding set is lost during a batch update." + "  This can create data leaks.");
                batch.add(result);
                if (batch.size() == 1000) {
                    try {
                        pcjStorage.addResults(pcjId, batch);
                    } catch (final PCJStorageException e) {
                        throw new TupleQueryResultHandlerException("Fail to batch load new results into the PCJ with ID '" + pcjId + "'.", e);
                    }
                    batch.clear();
                }
            }
        });
        if (!batch.isEmpty()) {
            pcjStorage.addResults(pcjId, batch);
            batch.clear();
        }
    } catch (final MalformedQueryException | PCJStorageException | SailException | QueryEvaluationException | RepositoryException | TupleQueryResultHandlerException e) {
        throw new RyaClientException("Fail to batch load new results into the PCJ with ID '" + pcjId + "'.", e);
    } finally {
        if (sailConn != null) {
            try {
                sailConn.close();
            } catch (final SailException e) {
                log.warn(e.getMessage(), e);
            }
        }
        if (sail != null) {
            try {
                sail.shutDown();
            } catch (final SailException e) {
                log.warn(e.getMessage(), e);
            }
        }
    }
}

Example 5 with MongoPcjStorage

use of org.apache.rya.indexing.pcj.storage.mongo.MongoPcjStorage in project incubator-rya by apache.

the class MongoCreatePCJ method createPCJ.

@Override
public String createPCJ(final String ryaInstanceName, final String sparql, final Set<ExportStrategy> strategies) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(ryaInstanceName);
    requireNonNull(sparql);
    // Ensure the Rya Instance exists.
    if (!instanceExists.exists(ryaInstanceName)) {
        throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", ryaInstanceName));
    }
    try (final MongoPcjStorage pcjStore = new MongoPcjStorage(mongoClient, ryaInstanceName)) {
        return pcjStore.createPcj(sparql);
    } catch (final PCJStorageException e) {
        throw new RyaClientException("Unable to create PCJ for: " + sparql, e);
    }
}