Search in sources :

Example 41 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloPcjStorageIT method addResults.

@Test
public void addResults() throws AccumuloException, AccumuloSecurityException, PCJStorageException, MalformedQueryException {
    // Setup the PCJ storage that will be tested against.
    final Connector connector = super.getClusterInstance().getConnector();
    final String ryaInstanceName = super.getRyaInstanceName();
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(connector, ryaInstanceName)) {
        // Create a PCJ.
        final String sparql = "SELECT * WHERE { ?a <http://isA> ?b }";
        final String pcjId = pcjStorage.createPcj(sparql);
        // Add some binding sets to it.
        final Set<VisibilityBindingSet> results = new HashSet<>();
        final MapBindingSet aliceBS = new MapBindingSet();
        aliceBS.addBinding("a", new URIImpl("http://Alice"));
        aliceBS.addBinding("b", new URIImpl("http://Person"));
        results.add(new VisibilityBindingSet(aliceBS, ""));
        final MapBindingSet charlieBS = new MapBindingSet();
        charlieBS.addBinding("a", new URIImpl("http://Charlie"));
        charlieBS.addBinding("b", new URIImpl("http://Comedian"));
        results.add(new VisibilityBindingSet(charlieBS, ""));
        pcjStorage.addResults(pcjId, results);
        // Make sure the PCJ metadata was updated.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        final Set<VariableOrder> varOrders = new ShiftVarOrderFactory().makeVarOrders(sparql);
        final PcjMetadata expectedMetadata = new PcjMetadata(sparql, 2L, varOrders);
        assertEquals(expectedMetadata, metadata);
    }
}
Also used : Connector(org.apache.accumulo.core.client.Connector) VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) AccumuloPcjStorage(org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage) VariableOrder(org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder) ShiftVarOrderFactory(org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory) URIImpl(org.openrdf.model.impl.URIImpl) PrecomputedJoinStorage(org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage) PcjMetadata(org.apache.rya.indexing.pcj.storage.PcjMetadata) MapBindingSet(org.openrdf.query.impl.MapBindingSet) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 42 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloPcjStorageIT method listResults.

@Test
public void listResults() throws Exception {
    // Setup the PCJ storage that will be tested against.
    final Connector connector = super.getClusterInstance().getConnector();
    final String ryaInstanceName = super.getRyaInstanceName();
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(connector, ryaInstanceName)) {
        // Create a PCJ.
        final String sparql = "SELECT * WHERE { ?a <http://isA> ?b }";
        final String pcjId = pcjStorage.createPcj(sparql);
        // Add some binding sets to it.
        final Set<VisibilityBindingSet> storedResults = new HashSet<>();
        final MapBindingSet aliceBS = new MapBindingSet();
        aliceBS.addBinding("a", new URIImpl("http://Alice"));
        aliceBS.addBinding("b", new URIImpl("http://Person"));
        storedResults.add(new VisibilityBindingSet(aliceBS, ""));
        final MapBindingSet charlieBS = new MapBindingSet();
        charlieBS.addBinding("a", new URIImpl("http://Charlie"));
        charlieBS.addBinding("b", new URIImpl("http://Comedian"));
        storedResults.add(new VisibilityBindingSet(charlieBS, ""));
        pcjStorage.addResults(pcjId, storedResults);
        // List the results that were stored.
        final Set<BindingSet> results = new HashSet<>();
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            while (resultsIt.hasNext()) {
                results.add(resultsIt.next());
            }
        }
        // The stored results are returned as normal binding sets, so unwrap them.
        final Set<BindingSet> expectedResults = storedResults.stream().map(visBs -> visBs.getBindingSet()).collect(Collectors.toSet());
        assertEquals(expectedResults, results);
    }
}
Also used : AccumuloRyaITBase(org.apache.rya.accumulo.AccumuloRyaITBase) PcjMetadata(org.apache.rya.indexing.pcj.storage.PcjMetadata) URIImpl(org.openrdf.model.impl.URIImpl) PCJDetails(org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.PCJDetails) NotInitializedException(org.apache.rya.api.instance.RyaDetailsRepository.NotInitializedException) ShiftVarOrderFactory(org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory) MapBindingSet(org.openrdf.query.impl.MapBindingSet) Connector(org.apache.accumulo.core.client.Connector) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) RyaDetailsRepository(org.apache.rya.api.instance.RyaDetailsRepository) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) BindingSet(org.openrdf.query.BindingSet) VariableOrder(org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder) MalformedQueryException(org.openrdf.query.MalformedQueryException) PCJStorageException(org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException) CloseableIterator(org.apache.rya.api.utils.CloseableIterator) ImmutableMap(com.google.common.collect.ImmutableMap) Assert.assertTrue(org.junit.Assert.assertTrue) Set(java.util.Set) Test(org.junit.Test) Collectors(java.util.stream.Collectors) AccumuloException(org.apache.accumulo.core.client.AccumuloException) PrecomputedJoinStorage(org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage) List(java.util.List) AccumuloRyaInstanceDetailsRepository(org.apache.rya.accumulo.instance.AccumuloRyaInstanceDetailsRepository) RyaDetailsRepositoryException(org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException) Assert.assertFalse(org.junit.Assert.assertFalse) AccumuloPcjStorage(org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) Connector(org.apache.accumulo.core.client.Connector) MapBindingSet(org.openrdf.query.impl.MapBindingSet) VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) BindingSet(org.openrdf.query.BindingSet) VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) AccumuloPcjStorage(org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage) URIImpl(org.openrdf.model.impl.URIImpl) PrecomputedJoinStorage(org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage) MapBindingSet(org.openrdf.query.impl.MapBindingSet) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 43 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloPcjStorageIT method purge.

@Test
public void purge() throws Exception {
    // Setup the PCJ storage that will be tested against.
    final Connector connector = super.getClusterInstance().getConnector();
    final String ryaInstanceName = super.getRyaInstanceName();
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(connector, ryaInstanceName)) {
        // Create a PCJ.
        final String sparql = "SELECT * WHERE { ?a <http://isA> ?b }";
        final String pcjId = pcjStorage.createPcj(sparql);
        // Add some binding sets to it.
        final Set<VisibilityBindingSet> expectedResults = new HashSet<>();
        final MapBindingSet aliceBS = new MapBindingSet();
        aliceBS.addBinding("a", new URIImpl("http://Alice"));
        aliceBS.addBinding("b", new URIImpl("http://Person"));
        expectedResults.add(new VisibilityBindingSet(aliceBS, ""));
        final MapBindingSet charlieBS = new MapBindingSet();
        charlieBS.addBinding("a", new URIImpl("http://Charlie"));
        charlieBS.addBinding("b", new URIImpl("http://Comedian"));
        expectedResults.add(new VisibilityBindingSet(charlieBS, ""));
        pcjStorage.addResults(pcjId, expectedResults);
        // Purge the PCJ.
        pcjStorage.purge(pcjId);
        // List the results that were stored.
        final Set<BindingSet> results = new HashSet<>();
        try (CloseableIterator<BindingSet> resultsIt = pcjStorage.listResults(pcjId)) {
            while (resultsIt.hasNext()) {
                results.add(resultsIt.next());
            }
        }
        assertTrue(results.isEmpty());
        // Make sure the PCJ metadata was updated.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        final Set<VariableOrder> varOrders = new ShiftVarOrderFactory().makeVarOrders(sparql);
        final PcjMetadata expectedMetadata = new PcjMetadata(sparql, 0L, varOrders);
        assertEquals(expectedMetadata, metadata);
    }
}
Also used : Connector(org.apache.accumulo.core.client.Connector) MapBindingSet(org.openrdf.query.impl.MapBindingSet) VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) BindingSet(org.openrdf.query.BindingSet) VisibilityBindingSet(org.apache.rya.api.model.VisibilityBindingSet) AccumuloPcjStorage(org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage) VariableOrder(org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder) ShiftVarOrderFactory(org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory) URIImpl(org.openrdf.model.impl.URIImpl) PrecomputedJoinStorage(org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage) PcjMetadata(org.apache.rya.indexing.pcj.storage.PcjMetadata) MapBindingSet(org.openrdf.query.impl.MapBindingSet) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 44 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class AccumuloPcjStorageIT method getPcjMetadata.

@Test
public void getPcjMetadata() throws AccumuloException, AccumuloSecurityException, PCJStorageException, MalformedQueryException {
    // Setup the PCJ storage that will be tested against.
    final Connector connector = super.getClusterInstance().getConnector();
    final String ryaInstanceName = super.getRyaInstanceName();
    try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(connector, ryaInstanceName)) {
        // Create a PCJ.
        final String sparql = "SELECT * WHERE { ?a <http://isA> ?b }";
        final String pcjId = pcjStorage.createPcj(sparql);
        // Fetch the PCJ's metadata.
        final PcjMetadata metadata = pcjStorage.getPcjMetadata(pcjId);
        // Ensure it has the expected values.
        final Set<VariableOrder> varOrders = new ShiftVarOrderFactory().makeVarOrders(sparql);
        final PcjMetadata expectedMetadata = new PcjMetadata(sparql, 0L, varOrders);
        assertEquals(expectedMetadata, metadata);
    }
}
Also used : Connector(org.apache.accumulo.core.client.Connector) AccumuloPcjStorage(org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage) PrecomputedJoinStorage(org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage) VariableOrder(org.apache.rya.indexing.pcj.storage.accumulo.VariableOrder) ShiftVarOrderFactory(org.apache.rya.indexing.pcj.storage.accumulo.ShiftVarOrderFactory) PcjMetadata(org.apache.rya.indexing.pcj.storage.PcjMetadata) Test(org.junit.Test)

Example 45 with AccumuloPcjStorage

use of org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage in project incubator-rya by apache.

the class RyaBindingSetExporterFactory method build.

@Override
public Optional<IncrementalResultExporter> build(final Context context) throws IncrementalExporterFactoryException, ConfigurationException {
    checkNotNull(context);
    // Wrap the context's parameters for parsing.
    final RyaExportParameters params = new RyaExportParameters(context.getObserverConfiguration().toMap());
    if (params.getUseRyaBindingSetExporter()) {
        // Setup Zookeeper connection info.
        final String accumuloInstance = params.getAccumuloInstanceName().get();
        final String zookeeperServers = params.getZookeeperServers().get().replaceAll(";", ",");
        final Instance inst = new ZooKeeperInstance(accumuloInstance, zookeeperServers);
        try {
            // Setup Accumulo connection info.
            final String exporterUsername = params.getExporterUsername().get();
            final String exporterPassword = params.getExporterPassword().get();
            final Connector accumuloConn = inst.getConnector(exporterUsername, new PasswordToken(exporterPassword));
            // Setup Rya PCJ Storage.
            final String ryaInstanceName = params.getRyaInstanceName().get();
            final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, ryaInstanceName);
            // Make the exporter.
            final IncrementalBindingSetExporter exporter = new RyaBindingSetExporter(pcjStorage);
            return Optional.of(exporter);
        } catch (final AccumuloException | AccumuloSecurityException e) {
            throw new IncrementalExporterFactoryException("Could not initialize the Accumulo connector using the provided configuration.", e);
        }
    } else {
        return Optional.absent();
    }
}
Also used : Connector(org.apache.accumulo.core.client.Connector) AccumuloException(org.apache.accumulo.core.client.AccumuloException) AccumuloPcjStorage(org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage) Instance(org.apache.accumulo.core.client.Instance) ZooKeeperInstance(org.apache.accumulo.core.client.ZooKeeperInstance) IncrementalBindingSetExporter(org.apache.rya.indexing.pcj.fluo.app.export.IncrementalBindingSetExporter) ZooKeeperInstance(org.apache.accumulo.core.client.ZooKeeperInstance) PasswordToken(org.apache.accumulo.core.client.security.tokens.PasswordToken) PrecomputedJoinStorage(org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException)

Aggregations

AccumuloPcjStorage (org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage)46 PrecomputedJoinStorage (org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage)44 Test (org.junit.Test)32 Connector (org.apache.accumulo.core.client.Connector)26 FluoClient (org.apache.fluo.api.client.FluoClient)21 CreateFluoPcj (org.apache.rya.indexing.pcj.fluo.api.CreateFluoPcj)20 BindingSet (org.openrdf.query.BindingSet)20 MapBindingSet (org.openrdf.query.impl.MapBindingSet)18 HashSet (java.util.HashSet)15 RyaStatement (org.apache.rya.api.domain.RyaStatement)15 ValueFactory (org.openrdf.model.ValueFactory)13 RyaURI (org.apache.rya.api.domain.RyaURI)12 InsertTriples (org.apache.rya.indexing.pcj.fluo.api.InsertTriples)11 PcjMetadata (org.apache.rya.indexing.pcj.storage.PcjMetadata)10 ValueFactoryImpl (org.openrdf.model.impl.ValueFactoryImpl)9 FluoClientImpl (org.apache.fluo.core.client.FluoClientImpl)7 VisibilityBindingSet (org.apache.rya.api.model.VisibilityBindingSet)7 SailRepositoryConnection (org.openrdf.repository.sail.SailRepositoryConnection)7 PCJDetails (org.apache.rya.api.instance.RyaDetails.PCJIndexDetails.PCJDetails)6 MalformedQueryException (org.openrdf.query.MalformedQueryException)6