use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.
the class PcjTablesIT method dropPcj.
@Test
public void dropPcj() throws PCJStorageException, AccumuloException, AccumuloSecurityException {
final Connector accumuloConn = cluster.getConnector();
// Create a PCJ index.
final String tableName = new PcjTableNameFactory().makeTableName(getRyaInstanceName(), "thePcj");
final Set<VariableOrder> varOrders = Sets.<VariableOrder>newHashSet(new VariableOrder("x"));
final String sparql = "SELECT x WHERE ?x <http://isA> <http://Food>";
final PcjTables pcjs = new PcjTables();
pcjs.createPcjTable(accumuloConn, tableName, varOrders, sparql);
// Fetch its metadata to show that it has actually been created.
final PcjMetadata expectedMetadata = new PcjMetadata(sparql, 0L, varOrders);
PcjMetadata metadata = pcjs.getPcjMetadata(accumuloConn, tableName);
assertEquals(expectedMetadata, metadata);
// Drop it.
pcjs.dropPcjTable(accumuloConn, tableName);
// Show the metadata is no longer present.
PCJStorageException tableDoesNotExistException = null;
try {
metadata = pcjs.getPcjMetadata(accumuloConn, tableName);
} catch (final PCJStorageException e) {
tableDoesNotExistException = e;
}
assertNotNull(tableDoesNotExistException);
}
use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.
the class MongoPcjDocuments method populatePcj.
/**
* Scan Rya for results that solve the PCJ's query and store them in the PCJ
* document.
* <p>
* This method assumes the PCJ document has already been created.
*
* @param pcjId - The Id of the PCJ that will receive the results. (not null)
* @param ryaConn - A connection to the Rya store that will be queried to find results. (not null)
* @throws PCJStorageException If results could not be written to the PCJ results document,
* the PCJ results document does not exist, or the query that is being execute was malformed.
*/
public void populatePcj(final String pcjId, final RepositoryConnection ryaConn) throws PCJStorageException {
checkNotNull(pcjId);
checkNotNull(ryaConn);
try {
// Fetch the query that needs to be executed from the PCJ metadata document.
final PcjMetadata pcjMetadata = getPcjMetadata(pcjId);
final String sparql = pcjMetadata.getSparql();
// Query Rya for results to the SPARQL query.
final TupleQuery query = ryaConn.prepareTupleQuery(QueryLanguage.SPARQL, sparql);
final TupleQueryResult results = query.evaluate();
// Load batches of 1000 of them at a time into the PCJ results document.
final Set<VisibilityBindingSet> batch = new HashSet<>(1000);
while (results.hasNext()) {
final VisibilityBindingSet bs = new VisibilityBindingSet(results.next());
batch.add(bs);
if (batch.size() == 1000) {
addResults(pcjId, batch);
batch.clear();
}
}
if (!batch.isEmpty()) {
addResults(pcjId, batch);
}
} catch (RepositoryException | MalformedQueryException | QueryEvaluationException e) {
throw new PCJStorageException("Could not populate a PCJ document with Rya results for the pcj with Id: " + pcjId, e);
}
}
use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.
the class PcjTables method getPcjMetadata.
/**
* Fetch the {@link PCJMetadata} from an Accumulo table.
* <p>
* This method assumes the PCJ table has already been created.
*
* @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
* @param pcjTableName - The name of the table that will be search. (not null)
* @return The PCJ Metadata that has been stolred in the in the PCJ Table.
* @throws PCJStorageException The PCJ Table does not exist.
*/
public PcjMetadata getPcjMetadata(final Connector accumuloConn, final String pcjTableName) throws PCJStorageException {
checkNotNull(accumuloConn);
checkNotNull(pcjTableName);
Scanner scanner = null;
try {
// Create an Accumulo scanner that iterates through the metadata entries.
scanner = accumuloConn.createScanner(pcjTableName, new Authorizations());
final Iterator<Entry<Key, Value>> entries = scanner.iterator();
// No metadata has been stored in the table yet.
if (!entries.hasNext()) {
throw new PCJStorageException("Could not find any PCJ metadata in the table named: " + pcjTableName);
}
// Fetch the metadata from the entries. Assuming they all have the same cardinality and sparql query.
String sparql = null;
Long cardinality = null;
final Set<VariableOrder> varOrders = new HashSet<>();
while (entries.hasNext()) {
final Entry<Key, Value> entry = entries.next();
final Text columnQualifier = entry.getKey().getColumnQualifier();
final byte[] value = entry.getValue().get();
if (columnQualifier.equals(PCJ_METADATA_SPARQL_QUERY)) {
sparql = stringLexicoder.decode(value);
} else if (columnQualifier.equals(PCJ_METADATA_CARDINALITY)) {
cardinality = longLexicoder.decode(value);
} else if (columnQualifier.equals(PCJ_METADATA_VARIABLE_ORDERS)) {
for (final String varOrderStr : listLexicoder.decode(value)) {
varOrders.add(new VariableOrder(varOrderStr));
}
}
}
return new PcjMetadata(sparql, cardinality, varOrders);
} catch (final TableNotFoundException e) {
throw new PCJStorageException("Could not add results to a PCJ because the PCJ table does not exist.", e);
} finally {
if (scanner != null) {
scanner.close();
}
}
}
use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.
the class PcjTables method createPcjTable.
/**
* Create a new PCJ table within an Accumulo instance for a SPARQL query.
* For example, calling the function like this:
* <pre>
* PcjTables.createPcjTable(
* accumuloConn,
*
* "foo_INDEX_query1234",
*
* Sets.newHashSet(
* new VariableOrder("city;worker;customer"),
* new VariableOrder("worker;customer;city") ,
* new VariableOrder("customer;city;worker")),
*
* "SELECT ?customer ?worker ?city { " +
* "?customer <http://talksTo> ?worker. " +
* "?worker <http://livesIn> ?city. " +
* "?worker <http://worksAt> <http://Home>. " +
* "}");
* </pre>
* </p>
* Will result in an Accumulo table named "foo_INDEX_query1234" with the following entries:
* <table border="1" style="width:100%">
* <tr> <th>Row ID</td> <th>Column</td> <th>Value</td> </tr>
* <tr> <td>pcjMetadata</td> <td>metadata:sparql</td> <td> ... UTF-8 bytes encoding the query string ... </td> </tr>
* <tr> <td>pcjMetadata</td> <td>metadata:cardinality</td> <td> The query's cardinality </td> </tr>
* <tr> <td>pcjMetadata</td> <td>metadata:variableOrders</td> <td> The variable orders the results are written to </td> </tr>
* </table>
*
* @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
* @param pcjTableName - The name of the table that will be created. (not null)
* @param varOrders - The variable orders the results within the table will be written to. (not null)
* @param sparql - The query this table's results solves. (not null)
* @throws PCJStorageException Could not create a new PCJ table either because Accumulo
* would not let us create it or the PCJ metadata was not able to be written to it.
*/
public void createPcjTable(final Connector accumuloConn, final String pcjTableName, final Set<VariableOrder> varOrders, final String sparql) throws PCJStorageException {
checkNotNull(accumuloConn);
checkNotNull(pcjTableName);
checkNotNull(varOrders);
checkNotNull(sparql);
final TableOperations tableOps = accumuloConn.tableOperations();
if (!tableOps.exists(pcjTableName)) {
BatchWriter writer = null;
try {
// Create the new table in Accumulo.
tableOps.create(pcjTableName);
// Write the PCJ Metadata to the newly created table.
final PcjMetadata pcjMetadata = new PcjMetadata(sparql, 0L, varOrders);
final List<Mutation> mutations = makeWriteMetadataMutations(pcjMetadata);
writer = accumuloConn.createBatchWriter(pcjTableName, new BatchWriterConfig());
writer.addMutations(mutations);
} catch (final TableExistsException e) {
log.warn("Something else just created the Rya PCJ export table named '" + pcjTableName + "'. This is unexpected, but we will continue as normal.");
} catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
throw new PCJStorageException("Could not create a new PCJ named: " + pcjTableName, e);
} finally {
if (writer != null) {
try {
writer.close();
} catch (final MutationsRejectedException e) {
log.error("Mutations rejected while creating the PCJ table.", e);
}
}
}
}
}
use of org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException in project incubator-rya by apache.
the class PcjTables method updateCardinality.
/**
* Update the cardinality of a PCJ by a {@code delta}.
*
* @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
* @param pcjTableName - The name of the PCJ table that will have its cardinality updated. (not null)
* @param delta - How much the cardinality will change.
* @throws PCJStorageException The cardinality could not be updated.
*/
private void updateCardinality(final Connector accumuloConn, final String pcjTableName, final long delta) throws PCJStorageException {
checkNotNull(accumuloConn);
checkNotNull(pcjTableName);
ConditionalWriter conditionalWriter = null;
try {
conditionalWriter = accumuloConn.createConditionalWriter(pcjTableName, new ConditionalWriterConfig());
boolean updated = false;
while (!updated) {
// Write the conditional update request to Accumulo.
final long cardinality = getPcjMetadata(accumuloConn, pcjTableName).getCardinality();
final ConditionalMutation mutation = makeUpdateCardinalityMutation(cardinality, delta);
final ConditionalWriter.Result result = conditionalWriter.write(mutation);
// Interpret the result.
switch(result.getStatus()) {
case ACCEPTED:
updated = true;
break;
case REJECTED:
break;
case UNKNOWN:
// We do not know if the mutation succeeded. At best, we
// can hope the metadata hasn't been updated
// since we originally fetched it and try again.
// Otherwise, continue forwards as if it worked. It's
// okay if this number is slightly off.
final long newCardinality = getPcjMetadata(accumuloConn, pcjTableName).getCardinality();
if (newCardinality != cardinality) {
updated = true;
}
break;
case VIOLATED:
throw new PCJStorageException("The cardinality could not be updated because the commit violated a table constraint.");
case INVISIBLE_VISIBILITY:
throw new PCJStorageException("The condition contains a visibility the updater can not satisfy.");
}
}
} catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
throw new PCJStorageException("Could not update the cardinality value of the PCJ Table named: " + pcjTableName, e);
} finally {
if (conditionalWriter != null) {
conditionalWriter.close();
}
}
}
Aggregations