Examples with MutationsRejectedException org.apache.accumulo.core.client.MutationsRejectedException used on opensource projects

Example 61 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class FinishedWorkUpdater method run.

@Override
public void run() {
    log.debug("Looking for finished replication work");
    if (!ReplicationTable.isOnline(conn)) {
        log.debug("Replication table is not yet online, will retry");
        return;
    }
    BatchScanner bs;
    BatchWriter replBw;
    try {
        bs = ReplicationTable.getBatchScanner(conn, 4);
        replBw = ReplicationTable.getBatchWriter(conn);
    } catch (ReplicationTableOfflineException e) {
        log.debug("Table is no longer online, will retry");
        return;
    }
    IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class);
    bs.addScanIterator(cfg);
    WorkSection.limit(bs);
    bs.setRanges(Collections.singleton(new Range()));
    try {
        for (Entry<Key, Value> serializedRow : bs) {
            SortedMap<Key, Value> wholeRow;
            try {
                wholeRow = WholeRowIterator.decodeRow(serializedRow.getKey(), serializedRow.getValue());
            } catch (IOException e) {
                log.warn("Could not deserialize whole row with key {}", serializedRow.getKey().toStringNoTruncate(), e);
                continue;
            }
            log.debug("Processing work progress for {} with {} columns", serializedRow.getKey().getRow(), wholeRow.size());
            Map<Table.ID, Long> tableIdToProgress = new HashMap<>();
            boolean error = false;
            Text buffer = new Text();
            // We want to determine what the minimum point that all Work entries have replicated to
            for (Entry<Key, Value> entry : wholeRow.entrySet()) {
                Status status;
                try {
                    status = Status.parseFrom(entry.getValue().get());
                } catch (InvalidProtocolBufferException e) {
                    log.warn("Could not deserialize protobuf for {}", entry.getKey(), e);
                    error = true;
                    break;
                }
                // Get the replication target for the work record
                entry.getKey().getColumnQualifier(buffer);
                ReplicationTarget target = ReplicationTarget.from(buffer);
                // Initialize the value in the map if we don't have one
                if (!tableIdToProgress.containsKey(target.getSourceTableId())) {
                    tableIdToProgress.put(target.getSourceTableId(), Long.MAX_VALUE);
                }
                // Find the minimum value for begin (everyone has replicated up to this offset in the file)
                tableIdToProgress.put(target.getSourceTableId(), Math.min(tableIdToProgress.get(target.getSourceTableId()), status.getBegin()));
            }
            if (error) {
                continue;
            }
            // Update the replication table for each source table we found work records for
            for (Entry<Table.ID, Long> entry : tableIdToProgress.entrySet()) {
                // If the progress is 0, then no one has replicated anything, and we don't need to update anything
                if (0 == entry.getValue()) {
                    continue;
                }
                serializedRow.getKey().getRow(buffer);
                log.debug("For {}, source table ID {} has replicated through {}", serializedRow.getKey().getRow(), entry.getKey(), entry.getValue());
                Mutation replMutation = new Mutation(buffer);
                // Set that we replicated at least this much data, ignoring the other fields
                Status updatedStatus = StatusUtil.replicated(entry.getValue());
                Value serializedUpdatedStatus = ProtobufUtil.toValue(updatedStatus);
                // Pull the sourceTableId into a Text
                Table.ID srcTableId = entry.getKey();
                // Make the mutation
                StatusSection.add(replMutation, srcTableId, serializedUpdatedStatus);
                log.debug("Updating replication status entry for {} with {}", serializedRow.getKey().getRow(), ProtobufUtil.toString(updatedStatus));
                try {
                    replBw.addMutation(replMutation);
                } catch (MutationsRejectedException e) {
                    log.error("Error writing mutations to update replication Status messages in StatusSection, will retry", e);
                    return;
                }
            }
        }
    } finally {
        log.debug("Finished updating files with completed replication work");
        bs.close();
        try {
            replBw.close();
        } catch (MutationsRejectedException e) {
            log.error("Error writing mutations to update replication Status messages in StatusSection, will retry", e);
        }
    }
}

Example 62 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class StatusMaker method addStatusRecord.

/**
 * Create a status record in the replication table
 */
protected boolean addStatusRecord(Text file, Table.ID tableId, Value v) {
    try {
        Mutation m = new Mutation(file);
        m.put(StatusSection.NAME, new Text(tableId.getUtf8()), v);
        try {
            replicationWriter.addMutation(m);
        } catch (MutationsRejectedException e) {
            log.warn("Failed to write work mutations for replication, will retry", e);
            return false;
        }
    } finally {
        try {
            replicationWriter.flush();
        } catch (MutationsRejectedException e) {
            log.warn("Failed to write work mutations for replication, will retry", e);
            return false;
        }
    }
    return true;
}

Example 63 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class ConstraintIT method test2.

private void test2(String table, boolean doFlush) throws Exception {
    // test sending multiple mutations with multiple constrain violations... all of the non violating mutations
    // should go through
    int numericErrors = 2;
    BatchWriter bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
    bw.addMutation(newMut("r1", "cf1", "cq1", "123"));
    bw.addMutation(newMut("r1", "cf1", "cq2", "I'm a bad value"));
    if (doFlush) {
        try {
            bw.flush();
            throw new Exception("Didn't find a bad mutation");
        } catch (MutationsRejectedException mre) {
            // ignored
            try {
                bw.close();
            } catch (MutationsRejectedException ex) {
            // ignored
            }
            bw = getConnector().createBatchWriter(table, new BatchWriterConfig());
            numericErrors = 1;
        }
    }
    bw.addMutation(newMut("r1", "cf1", "cq3", "I'm a naughty value"));
    bw.addMutation(newMut("@bad row@", "cf1", "cq2", "456"));
    bw.addMutation(newMut("r1", "cf1", "cq4", "789"));
    boolean sawMRE = false;
    try {
        bw.close();
        // should not get here
        throw new Exception("Test failed, constraint did not catch bad mutation");
    } catch (MutationsRejectedException mre) {
        System.out.println(mre);
        sawMRE = true;
        // verify constraint violation summary
        List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
        if (cvsl.size() != 2) {
            throw new Exception("Unexpected constraints");
        }
        HashMap<String, Integer> expected = new HashMap<>();
        expected.put("org.apache.accumulo.test.constraints.NumericValueConstraint", numericErrors);
        expected.put("org.apache.accumulo.test.constraints.AlphaNumKeyConstraint", 1);
        for (ConstraintViolationSummary cvs : cvsl) {
            if (expected.get(cvs.constrainClass) != cvs.numberOfViolatingMutations) {
                throw new Exception("Unexpected " + cvs.constrainClass + " " + cvs.numberOfViolatingMutations);
            }
        }
    }
    if (!sawMRE) {
        throw new Exception("Did not see MutationsRejectedException");
    }
    try (Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY)) {
        Iterator<Entry<Key, Value>> iter = scanner.iterator();
        Entry<Key, Value> entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq4")) || !entry.getValue().equals(new Value("789".getBytes(UTF_8)))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        if (iter.hasNext()) {
            entry = iter.next();
            throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
        }
    }
}

Example 64 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class ConstraintIT method test1.

private void test1(String tableName) throws Exception {
    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
    Mutation mut1 = new Mutation(new Text("r1"));
    mut1.put(new Text("cf1"), new Text("cq1"), new Value("123".getBytes(UTF_8)));
    bw.addMutation(mut1);
    // should not throw any exceptions
    bw.close();
    bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
    // create a mutation with a non numeric value
    Mutation mut2 = new Mutation(new Text("r1"));
    mut2.put(new Text("cf1"), new Text("cq1"), new Value("123a".getBytes(UTF_8)));
    bw.addMutation(mut2);
    boolean sawMRE = false;
    try {
        bw.close();
        // should not get here
        throw new Exception("Test failed, constraint did not catch bad mutation");
    } catch (MutationsRejectedException mre) {
        sawMRE = true;
        // verify constraint violation summary
        List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
        if (cvsl.size() != 1) {
            throw new Exception("Unexpected constraints");
        }
        for (ConstraintViolationSummary cvs : cvsl) {
            if (!cvs.constrainClass.equals(NumericValueConstraint.class.getName())) {
                throw new Exception("Unexpected constraint class " + cvs.constrainClass);
            }
            if (cvs.numberOfViolatingMutations != 1) {
                throw new Exception("Unexpected # violating mutations " + cvs.numberOfViolatingMutations);
            }
        }
    }
    if (!sawMRE) {
        throw new Exception("Did not see MutationsRejectedException");
    }
    // verify mutation did not go through
    try (Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY)) {
        scanner.setRange(new Range(new Text("r1")));
        Iterator<Entry<Key, Value>> iter = scanner.iterator();
        Entry<Key, Value> entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123".getBytes(UTF_8)))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        if (iter.hasNext()) {
            entry = iter.next();
            throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
        }
        // remove the numeric value constraint
        getConnector().tableOperations().removeConstraint(tableName, 2);
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        // now should be able to add a non numeric value
        bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
        bw.addMutation(mut2);
        bw.close();
        // verify mutation went through
        iter = scanner.iterator();
        entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        if (iter.hasNext()) {
            entry = iter.next();
            throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
        }
        // add a constraint that references a non-existant class
        getConnector().tableOperations().setProperty(tableName, Property.TABLE_CONSTRAINT_PREFIX + "1", "com.foobar.nonExistantClass");
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        // add a mutation
        bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
        Mutation mut3 = new Mutation(new Text("r1"));
        mut3.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
        bw.addMutation(mut3);
        sawMRE = false;
        try {
            bw.close();
            // should not get here
            throw new Exception("Test failed, mutation went through when table had bad constraints");
        } catch (MutationsRejectedException mre) {
            sawMRE = true;
        }
        if (!sawMRE) {
            throw new Exception("Did not see MutationsRejectedException");
        }
        // verify the mutation did not go through
        iter = scanner.iterator();
        entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a".getBytes(UTF_8)))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        if (iter.hasNext()) {
            entry = iter.next();
            throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
        }
        // remove the bad constraint
        getConnector().tableOperations().removeConstraint(tableName, 1);
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        // try the mutation again
        bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
        bw.addMutation(mut3);
        bw.close();
        // verify it went through
        iter = scanner.iterator();
        entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo".getBytes(UTF_8)))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        if (iter.hasNext()) {
            entry = iter.next();
            throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
        }
    }
}

Example 65 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class ProxyServer method closeWriter.

@Override
public void closeWriter(String writer) throws UnknownWriter, org.apache.accumulo.proxy.thrift.MutationsRejectedException, TException {
    try {
        BatchWriterPlusProblem bwpe = getWriter(writer);
        if (bwpe.exception != null)
            throw bwpe.exception;
        bwpe.writer.close();
        writerCache.invalidate(UUID.fromString(writer));
    } catch (UnknownWriter uw) {
        throw uw;
    } catch (MutationsRejectedException e) {
        throw new org.apache.accumulo.proxy.thrift.MutationsRejectedException(e.toString());
    } catch (Exception e) {
        throw new TException(e);
    }
}