Examples with MutationsRejectedException org.apache.accumulo.core.client.MutationsRejectedException used on opensource projects

Example 16 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project incubator-rya by apache.

the class PcjTables method createPcjTable.

/**
 * Create a new PCJ table within an Accumulo instance for a SPARQL query.
 * For example, calling the function like this:
 * <pre>
 * PcjTables.createPcjTable(
 *     accumuloConn,
 *
 *     "foo_INDEX_query1234",
 *
 *     Sets.newHashSet(
 *         new VariableOrder("city;worker;customer"),
 *         new VariableOrder("worker;customer;city") ,
 *         new VariableOrder("customer;city;worker")),
 *
 *     "SELECT ?customer ?worker ?city { " +
 *            "?customer &lt;http://talksTo> ?worker. " +
 *            "?worker &lt;http://livesIn> ?city. " +
 *            "?worker &lt;http://worksAt> &lt;http://Home>. " +
 *     "}");
 * </pre>
 * </p>
 * Will result in an Accumulo table named "foo_INDEX_query1234" with the following entries:
 * <table border="1" style="width:100%">
 *   <tr> <th>Row ID</td>  <th>Column</td>  <th>Value</td> </tr>
 *   <tr> <td>pcjMetadata</td> <td>metadata:sparql</td> <td> ... UTF-8 bytes encoding the query string ... </td> </tr>
 *   <tr> <td>pcjMetadata</td> <td>metadata:cardinality</td> <td> The query's cardinality </td> </tr>
 *   <tr> <td>pcjMetadata</td> <td>metadata:variableOrders</td> <td> The variable orders the results are written to </td> </tr>
 * </table>
 *
 * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
 * @param pcjTableName - The name of the table that will be created. (not null)
 * @param varOrders - The variable orders the results within the table will be written to. (not null)
 * @param sparql - The query this table's results solves. (not null)
 * @throws PCJStorageException Could not create a new PCJ table either because Accumulo
 *   would not let us create it or the PCJ metadata was not able to be written to it.
 */
public void createPcjTable(final Connector accumuloConn, final String pcjTableName, final Set<VariableOrder> varOrders, final String sparql) throws PCJStorageException {
    checkNotNull(accumuloConn);
    checkNotNull(pcjTableName);
    checkNotNull(varOrders);
    checkNotNull(sparql);
    final TableOperations tableOps = accumuloConn.tableOperations();
    if (!tableOps.exists(pcjTableName)) {
        BatchWriter writer = null;
        try {
            // Create the new table in Accumulo.
            tableOps.create(pcjTableName);
            // Write the PCJ Metadata to the newly created table.
            final PcjMetadata pcjMetadata = new PcjMetadata(sparql, 0L, varOrders);
            final List<Mutation> mutations = makeWriteMetadataMutations(pcjMetadata);
            writer = accumuloConn.createBatchWriter(pcjTableName, new BatchWriterConfig());
            writer.addMutations(mutations);
        } catch (final TableExistsException e) {
            log.warn("Something else just created the Rya PCJ export table named '" + pcjTableName + "'. This is unexpected, but we will continue as normal.");
        } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
            throw new PCJStorageException("Could not create a new PCJ named: " + pcjTableName, e);
        } finally {
            if (writer != null) {
                try {
                    writer.close();
                } catch (final MutationsRejectedException e) {
                    log.error("Mutations rejected while creating the PCJ table.", e);
                }
            }
        }
    }
}

Example 17 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project incubator-rya by apache.

the class PcjTables method writeResults.

/**
 * Add a collection of results to a specific PCJ table.
 *
 * @param accumuloConn - A connection to the Accumulo that hosts the PCJ table. (not null)
 * @param pcjTableName - The name of the PCJ table that will receive the results. (not null)
 * @param results - Binding sets that will be written to the PCJ table. (not null)
 * @throws PCJStorageException The provided PCJ table doesn't exist, is missing the
 *   PCJ metadata, or the result could not be written to it.
 */
private void writeResults(final Connector accumuloConn, final String pcjTableName, final Collection<VisibilityBindingSet> results) throws PCJStorageException {
    checkNotNull(accumuloConn);
    checkNotNull(pcjTableName);
    checkNotNull(results);
    // Fetch the variable orders from the PCJ table.
    final PcjMetadata metadata = getPcjMetadata(accumuloConn, pcjTableName);
    // Write each result formatted using each of the variable orders.
    BatchWriter writer = null;
    try {
        writer = accumuloConn.createBatchWriter(pcjTableName, new BatchWriterConfig());
        for (final VisibilityBindingSet result : results) {
            final Set<Mutation> addResultMutations = makeWriteResultMutations(metadata.getVarOrders(), result);
            writer.addMutations(addResultMutations);
        }
    } catch (TableNotFoundException | MutationsRejectedException e) {
        throw new PCJStorageException("Could not add results to the PCJ table named: " + pcjTableName, e);
    } finally {
        if (writer != null) {
            try {
                writer.close();
            } catch (final MutationsRejectedException e) {
                throw new PCJStorageException("Could not add results to a PCJ table because some of the mutations were rejected.", e);
            }
        }
    }
}

Example 18 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project incubator-rya by apache.

the class AccumuloFreeTextIndexer method deleteStatement.

private void deleteStatement(final Statement statement) throws IOException {
    Objects.requireNonNull(mtbw, "Freetext indexer attempting to delete, but setMultiTableBatchWriter() was not set.");
    // if the predicate list is empty, accept all predicates.
    // Otherwise, make sure the predicate is on the "valid" list
    final boolean isValidPredicate = validPredicates.isEmpty() || validPredicates.contains(statement.getPredicate());
    if (isValidPredicate && (statement.getObject() instanceof Literal)) {
        // Get the tokens
        final String text = statement.getObject().stringValue().toLowerCase();
        final SortedSet<String> tokens = tokenizer.tokenize(text);
        if (!tokens.isEmpty()) {
            // Get Document Data
            final String docContent = StatementSerializer.writeStatement(statement);
            final String docId = Md5Hash.md5Base64(docContent);
            // Setup partition
            final Text partition = genPartition(docContent.hashCode(), docTableNumPartitions);
            final Mutation docTableMut = new Mutation(partition);
            final List<Mutation> termTableMutations = new ArrayList<Mutation>();
            final Text docIdText = new Text(docId);
            // Delete the Document Data
            docTableMut.putDelete(ColumnPrefixes.DOCS_CF_PREFIX, docIdText);
            // Delete the statement parts in index
            docTableMut.putDelete(ColumnPrefixes.getSubjColFam(statement), docIdText);
            docTableMut.putDelete(ColumnPrefixes.getPredColFam(statement), docIdText);
            docTableMut.putDelete(ColumnPrefixes.getObjColFam(statement), docIdText);
            docTableMut.putDelete(ColumnPrefixes.getContextColFam(statement), docIdText);
            // Delete the statement terms in index
            for (final String token : tokens) {
                if (IS_TERM_TABLE_TOKEN_DELETION_ENABLED) {
                    final int rowId = Integer.parseInt(partition.toString());
                    final boolean doesTermExistInOtherDocs = doesTermExistInOtherDocs(token, rowId, docIdText);
                    // Only delete the term from the term table if it doesn't appear in other docs
                    if (!doesTermExistInOtherDocs) {
                        // Delete the term in the term table
                        termTableMutations.add(createEmptyPutDeleteMutation(ColumnPrefixes.getTermListColFam(token)));
                        termTableMutations.add(createEmptyPutDeleteMutation(ColumnPrefixes.getRevTermListColFam(token)));
                    }
                }
                // Un-tie the token to the document
                docTableMut.putDelete(ColumnPrefixes.getTermColFam(token), docIdText);
            }
            // write the mutations
            try {
                docTableBw.addMutation(docTableMut);
                termTableBw.addMutations(termTableMutations);
            } catch (final MutationsRejectedException e) {
                logger.error("error adding mutation", e);
                throw new IOException(e);
            }
        }
    }
}

Example 19 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class RemoveCompleteReplicationRecords method removeRowIfNecessary.

protected long removeRowIfNecessary(BatchWriter bw, SortedMap<Key, Value> columns, Text row, Text colf, Text colq) {
    long recordsRemoved = 0;
    if (columns.isEmpty()) {
        return recordsRemoved;
    }
    Mutation m = new Mutation(row);
    Map<Table.ID, Long> tableToTimeCreated = new HashMap<>();
    for (Entry<Key, Value> entry : columns.entrySet()) {
        Status status = null;
        try {
            status = Status.parseFrom(entry.getValue().get());
        } catch (InvalidProtocolBufferException e) {
            log.error("Encountered unparsable protobuf for key: {}", entry.getKey().toStringNoTruncate());
            continue;
        }
        // If a column in the row isn't ready for removal, we keep the whole row
        if (!StatusUtil.isSafeForRemoval(status)) {
            return 0l;
        }
        Key k = entry.getKey();
        k.getColumnFamily(colf);
        k.getColumnQualifier(colq);
        log.debug("Removing {} {}:{} from replication table", row, colf, colq);
        m.putDelete(colf, colq);
        Table.ID tableId;
        if (StatusSection.NAME.equals(colf)) {
            tableId = Table.ID.of(colq.toString());
        } else if (WorkSection.NAME.equals(colf)) {
            ReplicationTarget target = ReplicationTarget.from(colq);
            tableId = target.getSourceTableId();
        } else {
            throw new RuntimeException("Got unexpected column");
        }
        if (status.hasCreatedTime()) {
            Long timeClosed = tableToTimeCreated.get(tableId);
            if (null == timeClosed) {
                tableToTimeCreated.put(tableId, status.getCreatedTime());
            } else if (timeClosed != status.getCreatedTime()) {
                log.warn("Found multiple values for timeClosed for {}: {} and {}", row, timeClosed, status.getCreatedTime());
            }
        }
        recordsRemoved++;
    }
    List<Mutation> mutations = new ArrayList<>();
    mutations.add(m);
    for (Entry<Table.ID, Long> entry : tableToTimeCreated.entrySet()) {
        log.info("Removing order mutation for table {} at {} for {}", entry.getKey(), entry.getValue(), row.toString());
        Mutation orderMutation = OrderSection.createMutation(row.toString(), entry.getValue());
        orderMutation.putDelete(OrderSection.NAME, new Text(entry.getKey().getUtf8()));
        mutations.add(orderMutation);
    }
    // or not at all.
    try {
        bw.addMutations(mutations);
        bw.flush();
    } catch (MutationsRejectedException e) {
        log.error("Could not submit mutation to remove columns for {} in replication table", row, e);
        return 0l;
    }
    return recordsRemoved;
}

Example 20 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class RemoveCompleteReplicationRecords method run.

@Override
public void run() {
    BatchScanner bs;
    BatchWriter bw;
    try {
        bs = ReplicationTable.getBatchScanner(conn, 4);
        bw = ReplicationTable.getBatchWriter(conn);
        if (bs == null || bw == null)
            throw new AssertionError("Inconceivable; an exception should have been thrown, but 'bs' or 'bw' was null instead");
    } catch (ReplicationTableOfflineException e) {
        log.debug("Not attempting to remove complete replication records as the table ({}) isn't yet online", ReplicationTable.NAME);
        return;
    }
    bs.setRanges(Collections.singleton(new Range()));
    IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class);
    StatusSection.limit(bs);
    WorkSection.limit(bs);
    bs.addScanIterator(cfg);
    Stopwatch sw = new Stopwatch();
    long recordsRemoved = 0;
    try {
        sw.start();
        recordsRemoved = removeCompleteRecords(conn, bs, bw);
    } finally {
        if (null != bs) {
            bs.close();
        }
        if (null != bw) {
            try {
                bw.close();
            } catch (MutationsRejectedException e) {
                log.error("Error writing mutations to {}, will retry", ReplicationTable.NAME, e);
            }
        }
        sw.stop();
    }
    log.info("Removed {} complete replication entries from the table {}", recordsRemoved, ReplicationTable.NAME);
}