Examples with PrecomputedJoinStorage org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage used on opensource projects

Example 11 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class MongoPcjStorageIT method getPcjMetadata.

@Test
public void getPcjMetadata() throws Exception {
    try (final PrecomputedJoinStorage pcjStorage = new MongoPcjStorage(getMongoClient(), conf.getRyaInstanceName())) {
        final MongoRyaInstanceDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(getMongoClient(), conf.getRyaInstanceName());
        detailsRepo.initialize(RyaDetails.builder().setRyaInstanceName(conf.getRyaInstanceName()).setRyaVersion("test").setEntityCentricIndexDetails(new EntityCentricIndexDetails(false)).setTemporalIndexDetails(new TemporalIndexDetails(false)).setFreeTextDetails(new FreeTextIndexDetails(false)).setProspectorDetails(new ProspectorDetails(Optional.absent())).setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.absent())).setPCJIndexDetails(PCJIndexDetails.builder().setEnabled(true)).build());
        // Create a PCJ.
        final String sparql = "SELECT * WHERE { ?a <http://isA> ?b }";
        final String pcjId = pcjStorage.createPcj(sparql);
        // Fetch the PCJ's metadata.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        // Ensure it has the expected values.
        final Set<VariableOrder> varOrders = new ShiftVarOrderFactory().makeVarOrders(sparql);
        final PcjMetadata expectedMetadata = new PcjMetadata(sparql, 0L, varOrders);
        assertEquals(expectedMetadata, metadata);
    }
}

Example 12 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class GeoFunctionsIT method runTest.

public void runTest(final String sparql, final Collection<Statement> statements, final Collection<BindingSet> expectedResults) throws Exception {
    requireNonNull(sparql);
    requireNonNull(statements);
    requireNonNull(expectedResults);
    // Register the PCJ with Rya.
    final Instance accInstance = super.getAccumuloConnector().getInstance();
    final Connector accumuloConn = super.getAccumuloConnector();
    final RyaClient ryaClient = AccumuloRyaClientFactory.build(new AccumuloConnectionDetails(getUsername(), getPassword().toCharArray(), accInstance.getInstanceName(), accInstance.getZooKeepers()), accumuloConn);
    ryaClient.getCreatePCJ().createPCJ(getRyaInstanceName(), sparql);
    // Write the data to Rya.
    final SailRepositoryConnection ryaConn = super.getRyaSailRepository().getConnection();
    ryaConn.begin();
    ryaConn.add(statements);
    ryaConn.commit();
    ryaConn.close();
    // Wait for the Fluo application to finish computing the end result.
    super.getMiniFluo().waitForObservers();
    // Fetch the value that is stored within the PCJ table.
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName())) {
        final String pcjId = pcjStorage.listPcjs().get(0);
        final Set<BindingSet> results = Sets.newHashSet(pcjStorage.listResults(pcjId));
        // Ensure the result of the query matches the expected result.
        assertEquals(expectedResults, results);
    }
}

Example 13 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class AccumuloBatchUpdatePCJ method updatePCJResults.

private void updatePCJResults(final String ryaInstanceName, final String pcjId) throws InstanceDoesNotExistException, PCJDoesNotExistException, RyaClientException {
    // Things that have to be closed before we exit.
    Sail sail = null;
    SailConnection sailConn = null;
    CloseableIteration<? extends BindingSet, QueryEvaluationException> results = null;
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(super.getConnector(), ryaInstanceName)) {
        // Create an instance of Sail backed by the Rya instance.
        sail = connectToRya(ryaInstanceName);
        // Purge the old results from the PCJ.
        try {
            pcjStorage.purge(pcjId);
        } catch (final PCJStorageException e) {
            throw new RyaClientException("Could not batch update PCJ with ID '" + pcjId + "' because the old " + "results could not be purged from it.", e);
        }
        // Parse the PCJ's SPARQL query.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        final String sparql = metadata.getSparql();
        final SPARQLParser parser = new SPARQLParser();
        final ParsedQuery parsedQuery = parser.parseQuery(sparql, null);
        // Execute the query.
        sailConn = sail.getConnection();
        results = sailConn.evaluate(parsedQuery.getTupleExpr(), null, null, false);
        // Load the results into the PCJ table.
        final List<VisibilityBindingSet> batch = new ArrayList<>(1000);
        while (results.hasNext()) {
            final VisibilityBindingSet result = new VisibilityBindingSet(results.next(), "");
            batch.add(result);
            if (batch.size() == 1000) {
                pcjStorage.addResults(pcjId, batch);
                batch.clear();
            }
        }
        if (!batch.isEmpty()) {
            pcjStorage.addResults(pcjId, batch);
            batch.clear();
        }
    } catch (final MalformedQueryException | PCJStorageException | SailException | QueryEvaluationException e) {
        throw new RyaClientException("Fail to batch load new results into the PCJ with ID '" + pcjId + "'.", e);
    } finally {
        if (results != null) {
            try {
                results.close();
            } catch (final QueryEvaluationException e) {
                log.warn(e.getMessage(), e);
            }
        }
        if (sailConn != null) {
            try {
                sailConn.close();
            } catch (final SailException e) {
                log.warn(e.getMessage(), e);
            }
        }
        if (sail != null) {
            try {
                sail.shutDown();
            } catch (final SailException e) {
                log.warn(e.getMessage(), e);
            }
        }
    }
}

Example 14 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class AccumuloPcjStorageIT method createPCJ.

@Test
public void createPCJ() throws AccumuloException, AccumuloSecurityException, PCJStorageException, NotInitializedException, RyaDetailsRepositoryException {
    // Setup the PCJ storage that will be tested against.
    final Connector connector = super.getClusterInstance().getConnector();
    final String ryaInstanceName = super.getRyaInstanceName();
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(connector, ryaInstanceName)) {
        // Create a PCJ.
        final String pcjId = pcjStorage.createPcj("SELECT * WHERE { ?a <http://isA> ?b } ");
        // Ensure the Rya details have been updated to include the PCJ's ID.
        final RyaDetailsRepository detailsRepo = new AccumuloRyaInstanceDetailsRepository(connector, ryaInstanceName);
        final ImmutableMap<String, PCJDetails> detailsMap = detailsRepo.getRyaInstanceDetails().getPCJIndexDetails().getPCJDetails();
        final PCJDetails expectedDetails = PCJDetails.builder().setId(pcjId).build();
        assertEquals(expectedDetails, detailsMap.get(pcjId));
    }
}

Example 15 with PrecomputedJoinStorage

use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage in project incubator-rya by apache.

the class AccumuloPcjStorageIT method listPcjs.

@Test
public void listPcjs() throws AccumuloException, AccumuloSecurityException, PCJStorageException {
    // Setup the PCJ storage that will be tested against.
    final Connector connector = super.getClusterInstance().getConnector();
    final String ryaInstanceName = super.getRyaInstanceName();
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(connector, ryaInstanceName)) {
        // Create a few PCJs and hold onto their IDs.
        final List<String> expectedIds = new ArrayList<>();
        String pcjId = pcjStorage.createPcj("SELECT * WHERE { ?a <http://isA> ?b } ");
        expectedIds.add(pcjId);
        pcjId = pcjStorage.createPcj("SELECT * WHERE { ?a <http://isA> ?b } ");
        expectedIds.add(pcjId);
        pcjId = pcjStorage.createPcj("SELECT * WHERE { ?a <http://isA> ?b } ");
        expectedIds.add(pcjId);
        // Fetch the PCJ names
        final List<String> pcjIds = pcjStorage.listPcjs();
        // Ensure the expected IDs match the fetched IDs.
        Collections.sort(expectedIds);
        Collections.sort(pcjIds);
        assertEquals(expectedIds, pcjIds);
    }
}