Examples with MutationsRejectedException org.apache.accumulo.core.client.MutationsRejectedException used on opensource projects

Example 81 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class MultiTableBatchWriterIT method testOfflineTable.

@Test
public void testOfflineTable() throws Exception {
    boolean mutationsRejected = false;
    try {
        final String[] names = getUniqueNames(2);
        final String table1 = names[0], table2 = names[1];
        TableOperations tops = accumuloClient.tableOperations();
        tops.create(table1);
        tops.create(table2);
        BatchWriter bw1 = mtbw.getBatchWriter(table1), bw2 = mtbw.getBatchWriter(table2);
        Mutation m1 = new Mutation("foo");
        m1.put("col1", "", "val1");
        m1.put("col2", "", "val2");
        bw1.addMutation(m1);
        bw2.addMutation(m1);
        tops.offline(table1, true);
        tops.offline(table2, true);
        Mutation m2 = new Mutation("bar");
        m2.put("col1", "", "val1");
        m2.put("col2", "", "val2");
        try {
            bw1.addMutation(m2);
            bw2.addMutation(m2);
        } catch (MutationsRejectedException e) {
            // Pass -- Mutations might flush immediately and fail because of offline table
            mutationsRejected = true;
        }
    } finally {
        if (mtbw != null) {
            try {
                mtbw.close();
            } catch (MutationsRejectedException e) {
                // Pass
                mutationsRejected = true;
            }
        }
    }
    assertTrue("Expected mutations to be rejected.", mutationsRejected);
}

Example 82 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class BatchWriterReplicationReplayer method replicateLog.

@Override
public long replicateLog(ClientContext context, String tableName, WalEdits data) throws RemoteReplicationException {
    final LogFileKey key = new LogFileKey();
    final LogFileValue value = new LogFileValue();
    final long memoryInBytes = context.getConfiguration().getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY);
    BatchWriter bw = null;
    long mutationsApplied = 0L;
    try {
        for (ByteBuffer edit : data.getEdits()) {
            DataInputStream dis = new DataInputStream(ByteBufferUtil.toByteArrayInputStream(edit));
            try {
                key.readFields(dis);
                // TODO this is brittle because AccumuloReplicaSystem isn't actually calling
                // LogFileValue.write, but we're expecting
                // what we receive to be readable by the LogFileValue.
                value.readFields(dis);
            } catch (IOException e) {
                log.error("Could not deserialize edit from stream", e);
                throw new RemoteReplicationException(RemoteReplicationErrorCode.COULD_NOT_DESERIALIZE, "Could not deserialize edit from stream");
            }
            // Create the batchScanner if we don't already have one.
            if (bw == null) {
                BatchWriterConfig bwConfig = new BatchWriterConfig();
                bwConfig.setMaxMemory(memoryInBytes);
                try {
                    bw = context.createBatchWriter(tableName, bwConfig);
                } catch (TableNotFoundException e) {
                    throw new RemoteReplicationException(RemoteReplicationErrorCode.TABLE_DOES_NOT_EXIST, "Table " + tableName + " does not exist");
                }
            }
            log.info("Applying {} mutations to table {} as part of batch", value.mutations.size(), tableName);
            // If we got a ServerMutation, we have to make sure that we preserve the systemTimestamp
            // otherwise
            // the local system will assign a new timestamp.
            List<Mutation> mutationsCopy = new ArrayList<>(value.mutations.size());
            long mutationsCopied = 0L;
            for (Mutation orig : value.mutations) {
                if (orig instanceof ServerMutation) {
                    mutationsCopied++;
                    ServerMutation origServer = (ServerMutation) orig;
                    Mutation copy = new Mutation(orig.getRow());
                    for (ColumnUpdate update : orig.getUpdates()) {
                        long timestamp;
                        // If the update doesn't have a timestamp, pull it from the ServerMutation
                        if (update.hasTimestamp()) {
                            timestamp = update.getTimestamp();
                        } else {
                            timestamp = origServer.getSystemTimestamp();
                        }
                        // TODO ACCUMULO-2937 cache the CVs
                        if (update.isDeleted()) {
                            copy.putDelete(update.getColumnFamily(), update.getColumnQualifier(), new ColumnVisibility(update.getColumnVisibility()), timestamp);
                        } else {
                            copy.put(update.getColumnFamily(), update.getColumnQualifier(), new ColumnVisibility(update.getColumnVisibility()), timestamp, update.getValue());
                        }
                    }
                    // We also need to preserve the replicationSource information to prevent cycles
                    Set<String> replicationSources = orig.getReplicationSources();
                    if (replicationSources != null && !replicationSources.isEmpty()) {
                        for (String replicationSource : replicationSources) {
                            copy.addReplicationSource(replicationSource);
                        }
                    }
                    mutationsCopy.add(copy);
                } else {
                    mutationsCopy.add(orig);
                }
            }
            log.debug("Copied {} mutations to ensure server-assigned timestamps are propagated", mutationsCopied);
            try {
                bw.addMutations(mutationsCopy);
            } catch (MutationsRejectedException e) {
                log.error("Could not apply mutations to {}", tableName);
                throw new RemoteReplicationException(RemoteReplicationErrorCode.COULD_NOT_APPLY, "Could not apply mutations to " + tableName);
            }
            log.debug("{} mutations added to the BatchScanner", mutationsCopy.size());
            mutationsApplied += mutationsCopy.size();
        }
    } finally {
        if (bw != null) {
            try {
                bw.close();
            } catch (MutationsRejectedException e) {
                log.error("Could not apply mutations to {}", tableName);
                throw new RemoteReplicationException(RemoteReplicationErrorCode.COULD_NOT_APPLY, "Could not apply mutations to " + tableName);
            }
        }
    }
    log.info("Applied {} mutations in total to {}", mutationsApplied, tableName);
    return mutationsApplied;
}

Example 83 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class BatchWriterFlushIT method runMultiThreadedBinningTest.

@Test
public void runMultiThreadedBinningTest() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String[] tableNames = getUniqueNames(1);
        String tableName = tableNames[0];
        c.tableOperations().create(tableName);
        for (int x = 0; x < NUM_THREADS; x++) {
            c.tableOperations().addSplits(tableName, new TreeSet<>(Collections.singleton(new Text(Integer.toString(x * NUM_TO_FLUSH)))));
        }
        c.instanceOperations().waitForBalance();
        // Logger.getLogger(TabletServerBatchWriter.class).setLevel(Level.TRACE);
        final List<Set<Mutation>> allMuts = new LinkedList<>();
        List<Mutation> data = new ArrayList<>();
        for (int i = 0; i < NUM_THREADS; i++) {
            for (int j = 0; j < NUM_TO_FLUSH; j++) {
                int row = i * NUM_TO_FLUSH + j;
                Mutation m = new Mutation(new Text(String.format("%10d", row)));
                m.put("cf" + i, "cq", "" + row);
                data.add(m);
            }
        }
        assertEquals(NUM_THREADS * NUM_TO_FLUSH, data.size());
        Collections.shuffle(data);
        for (int n = 0; n < (NUM_THREADS * NUM_TO_FLUSH); n += NUM_TO_FLUSH) {
            Set<Mutation> muts = new HashSet<>(data.subList(n, n + NUM_TO_FLUSH));
            allMuts.add(muts);
        }
        ThreadPoolExecutor threads = ThreadPools.createFixedThreadPool(NUM_THREADS, "ClientThreads", false);
        threads.allowCoreThreadTimeOut(false);
        threads.prestartAllCoreThreads();
        BatchWriterConfig cfg = new BatchWriterConfig();
        cfg.setMaxLatency(10, TimeUnit.SECONDS);
        cfg.setMaxMemory(1 * 1024 * 1024);
        cfg.setMaxWriteThreads(NUM_THREADS);
        final BatchWriter bw = c.createBatchWriter(tableName, cfg);
        for (int k = 0; k < NUM_THREADS; k++) {
            final int idx = k;
            threads.execute(new Runnable() {

                @Override
                public void run() {
                    try {
                        bw.addMutations(allMuts.get(idx));
                        bw.flush();
                    } catch (MutationsRejectedException e) {
                        fail("Error adding mutations to batch writer");
                    }
                }
            });
        }
        threads.shutdown();
        threads.awaitTermination(3, TimeUnit.MINUTES);
        bw.close();
        try (Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY)) {
            for (Entry<Key, Value> e : scanner) {
                Mutation m = new Mutation(e.getKey().getRow());
                m.put(e.getKey().getColumnFamily(), e.getKey().getColumnQualifier(), e.getValue());
                boolean found = false;
                for (int l = 0; l < NUM_THREADS; l++) {
                    if (allMuts.get(l).contains(m)) {
                        found = true;
                        allMuts.get(l).remove(m);
                        break;
                    }
                }
                assertTrue("Mutation not found: " + m, found);
            }
            for (int m = 0; m < NUM_THREADS; m++) {
                assertEquals(0, allMuts.get(m).size());
            }
        }
    }
}

Example 84 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class DeleterFormatterTest method setUp.

@Before
public void setUp() throws IOException, MutationsRejectedException {
    input = new SettableInputStream();
    baos = new ByteArrayOutputStream();
    MutationsRejectedException mre = createMock(MutationsRejectedException.class);
    writer = createNiceMock(BatchWriter.class);
    exceptionWriter = createNiceMock(BatchWriter.class);
    exceptionWriter.close();
    expectLastCall().andThrow(mre);
    exceptionWriter.addMutation(anyObject(Mutation.class));
    expectLastCall().andThrow(mre);
    shellState = createNiceMock(Shell.class);
    terminal = new DumbTerminal(input, baos);
    terminal.setSize(new Size(80, 24));
    reader = LineReaderBuilder.builder().terminal(terminal).build();
    pw = terminal.writer();
    expect(shellState.getReader()).andReturn(reader).anyTimes();
    expect(shellState.getWriter()).andReturn(pw).anyTimes();
    replay(writer, exceptionWriter, shellState);
    data = new TreeMap<>();
    data.put(new Key("r", "cf", "cq"), new Value("value"));
}

Example 85 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class ConstraintIT method test1.

private void test1(AccumuloClient client, String tableName) throws Exception {
    BatchWriter bw = client.createBatchWriter(tableName);
    Mutation mut1 = new Mutation(new Text("r1"));
    mut1.put("cf1", "cq1", "123");
    bw.addMutation(mut1);
    // should not throw any exceptions
    bw.close();
    bw = client.createBatchWriter(tableName);
    // create a mutation with a non numeric value
    Mutation mut2 = new Mutation(new Text("r1"));
    mut2.put("cf1", "cq1", "123a");
    bw.addMutation(mut2);
    boolean sawMRE = false;
    try {
        bw.close();
        // should not get here
        throw new Exception("Test failed, constraint did not catch bad mutation");
    } catch (MutationsRejectedException mre) {
        sawMRE = true;
        // verify constraint violation summary
        List<ConstraintViolationSummary> cvsl = mre.getConstraintViolationSummaries();
        if (cvsl.size() != 1) {
            throw new Exception("Unexpected constraints");
        }
        for (ConstraintViolationSummary cvs : cvsl) {
            if (!cvs.constrainClass.equals(NumericValueConstraint.class.getName())) {
                throw new Exception("Unexpected constraint class " + cvs.constrainClass);
            }
            if (cvs.numberOfViolatingMutations != 1) {
                throw new Exception("Unexpected # violating mutations " + cvs.numberOfViolatingMutations);
            }
        }
    }
    if (!sawMRE) {
        throw new Exception("Did not see MutationsRejectedException");
    }
    // verify mutation did not go through
    try (Scanner scanner = client.createScanner(tableName, Authorizations.EMPTY)) {
        scanner.setRange(new Range(new Text("r1")));
        Iterator<Entry<Key, Value>> iter = scanner.iterator();
        Entry<Key, Value> entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123"))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        if (iter.hasNext()) {
            entry = iter.next();
            throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
        }
        // remove the numeric value constraint
        client.tableOperations().removeConstraint(tableName, 2);
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        // now should be able to add a non numeric value
        bw = client.createBatchWriter(tableName);
        bw.addMutation(mut2);
        bw.close();
        // verify mutation went through
        iter = scanner.iterator();
        entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a"))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        if (iter.hasNext()) {
            entry = iter.next();
            throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
        }
        // add a constraint that references a non-existent class
        client.tableOperations().setProperty(tableName, Property.TABLE_CONSTRAINT_PREFIX + "1", "com.foobar.nonExistentClass");
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        // add a mutation
        bw = client.createBatchWriter(tableName);
        Mutation mut3 = new Mutation(new Text("r1"));
        mut3.put("cf1", "cq1", "foo");
        bw.addMutation(mut3);
        sawMRE = false;
        try {
            bw.close();
            // should not get here
            throw new Exception("Test failed, mutation went through when table had bad constraints");
        } catch (MutationsRejectedException mre) {
            sawMRE = true;
        }
        if (!sawMRE) {
            throw new Exception("Did not see MutationsRejectedException");
        }
        // verify the mutation did not go through
        iter = scanner.iterator();
        entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("123a"))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        if (iter.hasNext()) {
            entry = iter.next();
            throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
        }
        // remove the bad constraint
        client.tableOperations().removeConstraint(tableName, 1);
        sleepUninterruptibly(1, TimeUnit.SECONDS);
        // try the mutation again
        bw = client.createBatchWriter(tableName);
        bw.addMutation(mut3);
        bw.close();
        // verify it went through
        iter = scanner.iterator();
        entry = iter.next();
        if (!entry.getKey().getRow().equals(new Text("r1")) || !entry.getKey().getColumnFamily().equals(new Text("cf1")) || !entry.getKey().getColumnQualifier().equals(new Text("cq1")) || !entry.getValue().equals(new Value("foo"))) {
            throw new Exception("Unexpected key or value " + entry.getKey() + " " + entry.getValue());
        }
        if (iter.hasNext()) {
            entry = iter.next();
            throw new Exception("Unexpected extra key or value " + entry.getKey() + " " + entry.getValue());
        }
    }
}