Examples with AccumuloPcjStorage org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage used on opensource projects

Example 6 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloDeletePCJIT method dropAndDestroyPCJ.

@Test
public void dropAndDestroyPCJ() throws InstanceDoesNotExistException, RyaClientException, PCJStorageException, RepositoryException, AccumuloException, AccumuloSecurityException, RyaDAOException {
    // Initialize the commands that will be used by this test.
    final CreatePCJ createPCJ = new AccumuloCreatePCJ(createConnectionDetails(), accumuloConn);
    // Create a PCJ.
    final String sparql1 = "SELECT ?x " + "WHERE { " + "?x <http://worksAt> <http://TacoJoint>." + "}";
    final String pcjId1 = createPCJ.createPCJ(getRyaInstanceName(), sparql1);
    // Create a PCJ.
    final String sparql2 = "SELECT ?x " + "WHERE { " + "?x <http://talksTo> <http://Eve>. " + "}";
    final String pcjId2 = createPCJ.createPCJ(getRyaInstanceName(), sparql2);
    // Verify a Query ID was added for the query within the Fluo app.
    List<String> fluoQueryIds = new ListQueryIds().listQueryIds(fluoClient);
    assertEquals(2, fluoQueryIds.size());
    // Insert some statements into Rya.
    final ValueFactory vf = ryaRepo.getValueFactory();
    ryaConn.add(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://Eve"));
    ryaConn.add(vf.createURI("http://Bob"), vf.createURI("http://talksTo"), vf.createURI("http://Eve"));
    ryaConn.add(vf.createURI("http://Charlie"), vf.createURI("http://talksTo"), vf.createURI("http://Eve"));
    ryaConn.add(vf.createURI("http://Eve"), vf.createURI("http://helps"), vf.createURI("http://Kevin"));
    ryaConn.add(vf.createURI("http://Bob"), vf.createURI("http://worksAt"), vf.createURI("http://TacoJoint"));
    ryaConn.add(vf.createURI("http://Charlie"), vf.createURI("http://worksAt"), vf.createURI("http://TacoJoint"));
    ryaConn.add(vf.createURI("http://Eve"), vf.createURI("http://worksAt"), vf.createURI("http://TacoJoint"));
    ryaConn.add(vf.createURI("http://David"), vf.createURI("http://worksAt"), vf.createURI("http://TacoJoint"));
    // Verify the correct results were exported.
    fluo.waitForObservers();
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName())) {
        assertEquals("the PCJ's metadata was added the storage.", 2, pcjStorage.listPcjs().size());
        // Delete all PCJ's.
        AccumuloRyaDAO dao = RyaSailFactory.getAccumuloDAOWithUpdatedConfig(conf);
        dao.dropAndDestroy();
        // Ensure the PCJ's metadata has been removed from the storage.
        assertTrue("the PCJ's metadata has been removed from the storage.", pcjStorage.listPcjs().isEmpty());
        // Ensure the PCJ has been removed from the Fluo application.
        fluo.waitForObservers();
    // Verify Query IDs were deleted for the query within the Fluo app.
    // TODO this fails, shows expected 0, but was 2.
    // fluoQueryIds = new ListQueryIds().listQueryIds(fluoClient);
    // assertEquals("Verify Query IDs were deleted for the query within the Fluo app.", 0, fluoQueryIds.size());
    }
}

Example 7 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloIndexSetProvider method getIndices.

@Override
protected List<ExternalTupleSet> getIndices() throws PcjIndexSetException {
    requireNonNull(conf);
    try {
        final String tablePrefix = requireNonNull(conf.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX));
        final Connector conn = requireNonNull(ConfigUtils.getConnector(conf));
        List<String> tables = null;
        if (conf instanceof RdfCloudTripleStoreConfiguration) {
            tables = ((RdfCloudTripleStoreConfiguration) conf).getPcjTables();
        }
        // this maps associates pcj table name with pcj sparql query
        final Map<String, String> indexTables = Maps.newLinkedHashMap();
        try (final PrecomputedJoinStorage storage = new AccumuloPcjStorage(conn, tablePrefix)) {
            final PcjTableNameFactory pcjFactory = new PcjTableNameFactory();
            final boolean tablesProvided = tables != null && !tables.isEmpty();
            if (tablesProvided) {
                // if tables provided, associate table name with sparql
                for (final String table : tables) {
                    indexTables.put(table, storage.getPcjMetadata(pcjFactory.getPcjId(table)).getSparql());
                }
            } else if (hasRyaDetails(tablePrefix, conn)) {
                // If this is a newer install of Rya, and it has PCJ Details, then
                // use those.
                final List<String> ids = storage.listPcjs();
                for (final String id : ids) {
                    indexTables.put(pcjFactory.makeTableName(tablePrefix, id), storage.getPcjMetadata(id).getSparql());
                }
            } else {
                // Otherwise figure it out by scanning tables.
                final PcjTables pcjTables = new PcjTables();
                for (final String table : conn.tableOperations().list()) {
                    if (table.startsWith(tablePrefix + "INDEX")) {
                        indexTables.put(table, pcjTables.getPcjMetadata(conn, table).getSparql());
                    }
                }
            }
        }
        // use table name sparql map (indexTables) to create {@link
        // AccumuloIndexSet}
        final List<ExternalTupleSet> index = Lists.newArrayList();
        if (indexTables.isEmpty()) {
            log.info("No Index found");
        } else {
            for (final String table : indexTables.keySet()) {
                final String indexSparqlString = indexTables.get(table);
                index.add(new AccumuloIndexSet(indexSparqlString, conf, table));
            }
        }
        return index;
    } catch (final PCJStorageException | AccumuloException | AccumuloSecurityException | MalformedQueryException | SailException | QueryEvaluationException | TableNotFoundException e) {
        throw new PcjIndexSetException("Failed to retrieve the indicies.", e);
    }
}

Example 8 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class GeoFunctionsIT method runTest.

public void runTest(final String sparql, final Collection<Statement> statements, final Collection<BindingSet> expectedResults) throws Exception {
    requireNonNull(sparql);
    requireNonNull(statements);
    requireNonNull(expectedResults);
    // Register the PCJ with Rya.
    final Instance accInstance = super.getAccumuloConnector().getInstance();
    final Connector accumuloConn = super.getAccumuloConnector();
    final RyaClient ryaClient = AccumuloRyaClientFactory.build(new AccumuloConnectionDetails(getUsername(), getPassword().toCharArray(), accInstance.getInstanceName(), accInstance.getZooKeepers()), accumuloConn);
    ryaClient.getCreatePCJ().createPCJ(getRyaInstanceName(), sparql);
    // Write the data to Rya.
    final SailRepositoryConnection ryaConn = super.getRyaSailRepository().getConnection();
    ryaConn.begin();
    ryaConn.add(statements);
    ryaConn.commit();
    ryaConn.close();
    // Wait for the Fluo application to finish computing the end result.
    super.getMiniFluo().waitForObservers();
    // Fetch the value that is stored within the PCJ table.
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName())) {
        final String pcjId = pcjStorage.listPcjs().get(0);
        final Set<BindingSet> results = Sets.newHashSet(pcjStorage.listResults(pcjId));
        // Ensure the result of the query matches the expected result.
        assertEquals(expectedResults, results);
    }
}

Example 9 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloBatchUpdatePCJ method updatePCJResults.

private void updatePCJResults(final String ryaInstanceName, final String pcjId) throws InstanceDoesNotExistException, PCJDoesNotExistException, RyaClientException {
    // Things that have to be closed before we exit.
    Sail sail = null;
    SailConnection sailConn = null;
    CloseableIteration<? extends BindingSet, QueryEvaluationException> results = null;
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(super.getConnector(), ryaInstanceName)) {
        // Create an instance of Sail backed by the Rya instance.
        sail = connectToRya(ryaInstanceName);
        // Purge the old results from the PCJ.
        try {
            pcjStorage.purge(pcjId);
        } catch (final PCJStorageException e) {
            throw new RyaClientException("Could not batch update PCJ with ID '" + pcjId + "' because the old " + "results could not be purged from it.", e);
        }
        // Parse the PCJ's SPARQL query.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        final String sparql = metadata.getSparql();
        final SPARQLParser parser = new SPARQLParser();
        final ParsedQuery parsedQuery = parser.parseQuery(sparql, null);
        // Execute the query.
        sailConn = sail.getConnection();
        results = sailConn.evaluate(parsedQuery.getTupleExpr(), null, null, false);
        // Load the results into the PCJ table.
        final List<VisibilityBindingSet> batch = new ArrayList<>(1000);
        while (results.hasNext()) {
            final VisibilityBindingSet result = new VisibilityBindingSet(results.next(), "");
            batch.add(result);
            if (batch.size() == 1000) {
                pcjStorage.addResults(pcjId, batch);
                batch.clear();
            }
        }
        if (!batch.isEmpty()) {
            pcjStorage.addResults(pcjId, batch);
            batch.clear();
        }
    } catch (final MalformedQueryException | PCJStorageException | SailException | QueryEvaluationException e) {
        throw new RyaClientException("Fail to batch load new results into the PCJ with ID '" + pcjId + "'.", e);
    } finally {
        if (results != null) {
            try {
                results.close();
            } catch (final QueryEvaluationException e) {
                log.warn(e.getMessage(), e);
            }
        }
        if (sailConn != null) {
            try {
                sailConn.close();
            } catch (final SailException e) {
                log.warn(e.getMessage(), e);
            }
        }
        if (sail != null) {
            try {
                sail.shutDown();
            } catch (final SailException e) {
                log.warn(e.getMessage(), e);
            }
        }
    }
}

Example 10 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloPcjStorageIT method createPCJ.

@Test
public void createPCJ() throws AccumuloException, AccumuloSecurityException, PCJStorageException, NotInitializedException, RyaDetailsRepositoryException {
    // Setup the PCJ storage that will be tested against.
    final Connector connector = super.getClusterInstance().getConnector();
    final String ryaInstanceName = super.getRyaInstanceName();
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(connector, ryaInstanceName)) {
        // Create a PCJ.
        final String pcjId = pcjStorage.createPcj("SELECT * WHERE { ?a <http://isA> ?b } ");
        // Ensure the Rya details have been updated to include the PCJ's ID.
        final RyaDetailsRepository detailsRepo = new AccumuloRyaInstanceDetailsRepository(connector, ryaInstanceName);
        final ImmutableMap<String, PCJDetails> detailsMap = detailsRepo.getRyaInstanceDetails().getPCJIndexDetails().getPCJDetails();
        final PCJDetails expectedDetails = PCJDetails.builder().setId(pcjId).build();
        assertEquals(expectedDetails, detailsMap.get(pcjId));
    }
}