Examples with AccumuloPcjStorage org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage used on opensource projects

Example 1 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class PrecomputedJoinStorageSupplierTest method configuredForAccumulo.

@Test
public void configuredForAccumulo() {
    // Create a supplier that does not return any configuration.
    final Supplier<Configuration> configSupplier = mock(Supplier.class);
    final Configuration config = new Configuration();
    config.set(PrecomputedJoinIndexerConfig.PCJ_STORAGE_TYPE, PrecomputedJoinStorageType.ACCUMULO.toString());
    when(configSupplier.get()).thenReturn(config);
    final AccumuloPcjStorageSupplier accumuloSupplier = mock(AccumuloPcjStorageSupplier.class);
    final AccumuloPcjStorage mockAccumuloStorage = mock(AccumuloPcjStorage.class);
    when(accumuloSupplier.get()).thenReturn(mockAccumuloStorage);
    final PrecomputedJoinStorageSupplier storageSupplier = new PrecomputedJoinStorageSupplier(configSupplier, accumuloSupplier);
    // Ensure the mock AccumuloPcjStorage is what was returned.
    final PrecomputedJoinStorage storage = storageSupplier.get();
    assertEquals(mockAccumuloStorage, storage);
}

Example 2 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloCreatePCJ method createPCJ.

@Override
public String createPCJ(final String instanceName, final String sparql, Set<ExportStrategy> strategies) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(instanceName);
    requireNonNull(sparql);
    final Optional<RyaDetails> ryaDetailsHolder = getInstanceDetails.getDetails(instanceName);
    final boolean ryaInstanceExists = ryaDetailsHolder.isPresent();
    if (!ryaInstanceExists) {
        throw new InstanceDoesNotExistException(String.format("The '%s' instance of Rya does not exist.", instanceName));
    }
    final PCJIndexDetails pcjIndexDetails = ryaDetailsHolder.get().getPCJIndexDetails();
    final boolean pcjIndexingEnabeld = pcjIndexDetails.isEnabled();
    if (!pcjIndexingEnabeld) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ Indexing enabled.", instanceName));
    }
    // Create the PCJ table that will receive the index results.
    final String pcjId;
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(getConnector(), instanceName)) {
        pcjId = pcjStorage.createPcj(sparql);
        // If a Fluo application is being used, task it with updating the PCJ.
        final Optional<FluoDetails> fluoDetailsHolder = pcjIndexDetails.getFluoDetails();
        if (fluoDetailsHolder.isPresent()) {
            final String fluoAppName = fluoDetailsHolder.get().getUpdateAppName();
            try {
                updateFluoApp(instanceName, fluoAppName, pcjId, sparql, strategies);
            } catch (RepositoryException | MalformedQueryException | SailException | QueryEvaluationException | PcjException | RyaDAOException e) {
                throw new RyaClientException("Problem while initializing the Fluo application with the new PCJ.", e);
            } catch (UnsupportedQueryException e) {
                throw new RyaClientException("The new PCJ could not be initialized because it either contains an unsupported query node " + "or an invalid ExportStrategy for the given QueryType.  Projection queries can be exported to either Rya or Kafka," + "unless they contain an aggregation, in which case they can only be exported to Kafka.  Construct queries can be exported" + "to Rya and Kafka, and Periodic queries can only be exported to Rya.");
            }
            // Update the Rya Details to indicate the PCJ is being updated incrementally.
            final RyaDetailsRepository detailsRepo = new AccumuloRyaInstanceDetailsRepository(getConnector(), instanceName);
            try {
                new RyaDetailsUpdater(detailsRepo).update(new RyaDetailsMutator() {

                    @Override
                    public RyaDetails mutate(final RyaDetails originalDetails) throws CouldNotApplyMutationException {
                        // Update the original PCJ Details to indicate they are incrementally updated.
                        final PCJDetails originalPCJDetails = originalDetails.getPCJIndexDetails().getPCJDetails().get(pcjId);
                        final PCJDetails.Builder mutatedPCJDetails = PCJDetails.builder(originalPCJDetails).setUpdateStrategy(PCJUpdateStrategy.INCREMENTAL);
                        // Replace the old PCJ Details with the updated ones.
                        final RyaDetails.Builder builder = RyaDetails.builder(originalDetails);
                        builder.getPCJIndexDetails().addPCJDetails(mutatedPCJDetails);
                        return builder.build();
                    }
                });
            } catch (RyaDetailsRepositoryException | CouldNotApplyMutationException e) {
                throw new RyaClientException("Problem while updating the Rya instance's Details to indicate the PCJ is being incrementally updated.", e);
            }
        }
        // Return the ID that was assigned to the PCJ.
        return pcjId;
    } catch (final PCJStorageException e) {
        throw new RyaClientException("Problem while initializing the PCJ table.", e);
    }
}

Example 3 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloDeletePCJ method deletePCJ.

@Override
public void deletePCJ(final String instanceName, final String pcjId) throws InstanceDoesNotExistException, RyaClientException {
    requireNonNull(instanceName);
    requireNonNull(pcjId);
    final Optional<RyaDetails> originalDetails = getInstanceDetails.getDetails(instanceName);
    final boolean ryaInstanceExists = originalDetails.isPresent();
    if (!ryaInstanceExists) {
        throw new InstanceDoesNotExistException(String.format("The '%s' instance of Rya does not exist.", instanceName));
    }
    final boolean pcjIndexingEnabeld = originalDetails.get().getPCJIndexDetails().isEnabled();
    if (!pcjIndexingEnabeld) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ Indexing enabled.", instanceName));
    }
    final boolean pcjExists = originalDetails.get().getPCJIndexDetails().getPCJDetails().containsKey(pcjId);
    if (!pcjExists) {
        throw new RyaClientException(String.format("The '%s' instance of Rya does not have PCJ with ID '%s'.", instanceName, pcjId));
    }
    // If the PCJ was being maintained by a Fluo application, then stop that process.
    final PCJIndexDetails pcjIndexDetails = originalDetails.get().getPCJIndexDetails();
    final PCJDetails droppedPcjDetails = pcjIndexDetails.getPCJDetails().get(pcjId);
    if (droppedPcjDetails.getUpdateStrategy().isPresent()) {
        if (droppedPcjDetails.getUpdateStrategy().get() == PCJUpdateStrategy.INCREMENTAL) {
            final Optional<FluoDetails> fluoDetailsHolder = pcjIndexDetails.getFluoDetails();
            if (fluoDetailsHolder.isPresent()) {
                final String fluoAppName = pcjIndexDetails.getFluoDetails().get().getUpdateAppName();
                stopUpdatingPCJ(fluoAppName, pcjId);
            } else {
                log.error(String.format("Could not stop the Fluo application from updating the PCJ because the Fluo Details are " + "missing for the Rya instance named '%s'.", instanceName));
            }
        }
    }
    // Drop the table that holds the PCJ results from Accumulo.
    try (final PrecomputedJoinStorage pcjs = new AccumuloPcjStorage(getConnector(), instanceName)) {
        pcjs.dropPcj(pcjId);
    } catch (final PCJStorageException e) {
        throw new RyaClientException("Could not drop the PCJ's table from Accumulo.", e);
    }
}

Example 4 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloBatchUpdatePCJIT method batchUpdate.

@Test
public void batchUpdate() throws Exception {
    // Setup a Rya Client.
    final AccumuloConnectionDetails connectionDetails = new AccumuloConnectionDetails(super.getUsername(), super.getPassword().toCharArray(), super.getInstanceName(), super.getZookeepers());
    final RyaClient ryaClient = AccumuloRyaClientFactory.build(connectionDetails, super.getConnector());
    // Install an instance of Rya on the mini accumulo cluster.
    ryaClient.getInstall().install(RYA_INSTANCE_NAME, InstallConfiguration.builder().setEnablePcjIndex(true).build());
    Sail sail = null;
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(super.getConnector(), RYA_INSTANCE_NAME)) {
        // Get a Sail connection backed by the installed Rya instance.
        final AccumuloRdfConfiguration ryaConf = new AccumuloRdfConfiguration();
        ryaConf.setTablePrefix(RYA_INSTANCE_NAME);
        ryaConf.set(ConfigUtils.CLOUDBASE_USER, super.getUsername());
        ryaConf.set(ConfigUtils.CLOUDBASE_PASSWORD, super.getPassword());
        ryaConf.set(ConfigUtils.CLOUDBASE_ZOOKEEPERS, super.getZookeepers());
        ryaConf.set(ConfigUtils.CLOUDBASE_INSTANCE, super.getInstanceName());
        ryaConf.set(ConfigUtils.USE_PCJ, "true");
        ryaConf.set(ConfigUtils.PCJ_STORAGE_TYPE, PrecomputedJoinStorageType.ACCUMULO.toString());
        ryaConf.set(ConfigUtils.PCJ_UPDATER_TYPE, PrecomputedJoinUpdaterType.NO_UPDATE.toString());
        sail = RyaSailFactory.getInstance(ryaConf);
        // Load some statements into the Rya instance.
        final ValueFactory vf = sail.getValueFactory();
        final SailConnection sailConn = sail.getConnection();
        sailConn.begin();
        sailConn.addStatement(vf.createURI("urn:Alice"), vf.createURI("urn:likes"), vf.createURI("urn:icecream"));
        sailConn.addStatement(vf.createURI("urn:Bob"), vf.createURI("urn:likes"), vf.createURI("urn:icecream"));
        sailConn.addStatement(vf.createURI("urn:Charlie"), vf.createURI("urn:likes"), vf.createURI("urn:icecream"));
        sailConn.addStatement(vf.createURI("urn:David"), vf.createURI("urn:likes"), vf.createURI("urn:icecream"));
        sailConn.addStatement(vf.createURI("urn:Eve"), vf.createURI("urn:likes"), vf.createURI("urn:icecream"));
        sailConn.addStatement(vf.createURI("urn:Frank"), vf.createURI("urn:likes"), vf.createURI("urn:icecream"));
        sailConn.addStatement(vf.createURI("urn:George"), vf.createURI("urn:likes"), vf.createURI("urn:icecream"));
        sailConn.addStatement(vf.createURI("urn:Hillary"), vf.createURI("urn:likes"), vf.createURI("urn:icecream"));
        sailConn.addStatement(vf.createURI("urn:Alice"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue"));
        sailConn.addStatement(vf.createURI("urn:Bob"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue"));
        sailConn.addStatement(vf.createURI("urn:Charlie"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue"));
        sailConn.addStatement(vf.createURI("urn:David"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue"));
        sailConn.addStatement(vf.createURI("urn:Eve"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue"));
        sailConn.addStatement(vf.createURI("urn:Frank"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:blue"));
        sailConn.addStatement(vf.createURI("urn:George"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:green"));
        sailConn.addStatement(vf.createURI("urn:Hillary"), vf.createURI("urn:hasEyeColor"), vf.createURI("urn:brown"));
        sailConn.commit();
        sailConn.close();
        // Create a PCJ for a SPARQL query.
        final String sparql = "SELECT ?name WHERE { ?name <urn:likes> <urn:icecream> . ?name <urn:hasEyeColor> <urn:blue> . }";
        final String pcjId = pcjStorage.createPcj(sparql);
        // Run the test.
        ryaClient.getBatchUpdatePCJ().batchUpdate(RYA_INSTANCE_NAME, pcjId);
        // Verify the correct results were loaded into the PCJ table.
        final Set<BindingSet> expectedResults = new HashSet<>();
        MapBindingSet bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Alice"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Bob"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Charlie"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:David"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Eve"));
        expectedResults.add(bs);
        bs = new MapBindingSet();
        bs.addBinding("name", vf.createURI("urn:Frank"));
        expectedResults.add(bs);
        final Set<BindingSet> results = new HashSet<>();
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            while (resultsIt.hasNext()) {
                results.add(resultsIt.next());
            }
        }
        assertEquals(expectedResults, results);
    } finally {
        if (sail != null) {
            sail.shutDown();
        }
    }
}

Example 5 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloDeletePCJIT method deletePCJ.

@Test
public void deletePCJ() throws InstanceDoesNotExistException, RyaClientException, PCJStorageException, RepositoryException {
    // Initialize the commands that will be used by this test.
    final CreatePCJ createPCJ = new AccumuloCreatePCJ(createConnectionDetails(), accumuloConn);
    // Create a PCJ.
    final String sparql = "SELECT ?x " + "WHERE { " + "?x <http://talksTo> <http://Eve>. " + "?x <http://worksAt> <http://TacoJoint>." + "}";
    final String pcjId = createPCJ.createPCJ(getRyaInstanceName(), sparql);
    // Verify a Query ID was added for the query within the Fluo app.
    List<String> fluoQueryIds = new ListQueryIds().listQueryIds(fluoClient);
    assertEquals(1, fluoQueryIds.size());
    // Insert some statements into Rya.
    final ValueFactory vf = ryaRepo.getValueFactory();
    ryaConn.add(vf.createURI("http://Alice"), vf.createURI("http://talksTo"), vf.createURI("http://Eve"));
    ryaConn.add(vf.createURI("http://Bob"), vf.createURI("http://talksTo"), vf.createURI("http://Eve"));
    ryaConn.add(vf.createURI("http://Charlie"), vf.createURI("http://talksTo"), vf.createURI("http://Eve"));
    ryaConn.add(vf.createURI("http://Eve"), vf.createURI("http://helps"), vf.createURI("http://Kevin"));
    ryaConn.add(vf.createURI("http://Bob"), vf.createURI("http://worksAt"), vf.createURI("http://TacoJoint"));
    ryaConn.add(vf.createURI("http://Charlie"), vf.createURI("http://worksAt"), vf.createURI("http://TacoJoint"));
    ryaConn.add(vf.createURI("http://Eve"), vf.createURI("http://worksAt"), vf.createURI("http://TacoJoint"));
    ryaConn.add(vf.createURI("http://David"), vf.createURI("http://worksAt"), vf.createURI("http://TacoJoint"));
    // Verify the correct results were exported.
    fluo.waitForObservers();
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName())) {
        final Set<BindingSet> results = Sets.newHashSet(pcjStorage.listResults(pcjId));
        final MapBindingSet bob = new MapBindingSet();
        bob.addBinding("x", vf.createURI("http://Bob"));
        final MapBindingSet charlie = new MapBindingSet();
        charlie.addBinding("x", vf.createURI("http://Charlie"));
        final Set<BindingSet> expected = Sets.<BindingSet>newHashSet(bob, charlie);
        assertEquals(expected, results);
        // Delete the PCJ.
        final DeletePCJ deletePCJ = new AccumuloDeletePCJ(createConnectionDetails(), accumuloConn);
        deletePCJ.deletePCJ(getRyaInstanceName(), pcjId);
        // Ensure the PCJ's metadata has been removed from the storage.
        assertTrue(pcjStorage.listPcjs().isEmpty());
        // Ensure the PCJ has been removed from the Fluo application.
        fluo.waitForObservers();
        // Verify a Query ID was added for the query within the Fluo app.
        fluoQueryIds = new ListQueryIds().listQueryIds(fluoClient);
        assertEquals(0, fluoQueryIds.size());
    }
}