Examples with MutationsRejectedException org.apache.accumulo.core.client.MutationsRejectedException used on opensource projects

Example 1 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project YCSB by brianfrankcooper.

the class AccumuloClient method getTable.

/**
   * Called when the user specifies a table that isn't the same as the existing
   * table. Connect to it and if necessary, close our current connection.
   * 
   * @param t
   *          The table to open.
   */
public void getTable(String t) throws TableNotFoundException {
    if (bw != null) {
        // Close the existing writer if necessary.
        try {
            bw.close();
        } catch (MutationsRejectedException e) {
            // Couldn't spit out the mutations we wanted.
            // Ignore this for now.
            System.err.println("MutationsRejectedException: " + e.getMessage());
        }
    }
    BatchWriterConfig bwc = new BatchWriterConfig();
    bwc.setMaxLatency(Long.parseLong(getProperties().getProperty("accumulo.batchWriterMaxLatency", "30000")), TimeUnit.MILLISECONDS);
    bwc.setMaxMemory(Long.parseLong(getProperties().getProperty("accumulo.batchWriterSize", "100000")));
    bwc.setMaxWriteThreads(Integer.parseInt(getProperties().getProperty("accumulo.batchWriterThreads", "1")));
    bw = connector.createBatchWriter(t, bwc);
    // Create our scanners
    singleScanner = connector.createScanner(t, Authorizations.EMPTY);
    scanScanner = connector.createScanner(t, Authorizations.EMPTY);
    // Store the name of the table we have open.
    table = t;
}

Example 2 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project Gaffer by gchq.

the class AccumuloStore method insertGraphElements.

protected void insertGraphElements(final Iterable<Element> elements) throws StoreException {
    // Create BatchWriter
    final BatchWriter writer = TableUtils.createBatchWriter(this);
    // too high a latency, etc.
    if (elements != null) {
        for (final Element element : elements) {
            final Pair<Key> keys;
            try {
                keys = keyPackage.getKeyConverter().getKeysFromElement(element);
            } catch (final AccumuloElementConversionException e) {
                LOGGER.error("Failed to create an accumulo key from element of type " + element.getGroup() + " when trying to insert elements");
                continue;
            }
            final Value value;
            try {
                value = keyPackage.getKeyConverter().getValueFromElement(element);
            } catch (final AccumuloElementConversionException e) {
                LOGGER.error("Failed to create an accumulo value from element of type " + element.getGroup() + " when trying to insert elements");
                continue;
            }
            final Mutation m = new Mutation(keys.getFirst().getRow());
            m.put(keys.getFirst().getColumnFamily(), keys.getFirst().getColumnQualifier(), new ColumnVisibility(keys.getFirst().getColumnVisibility()), keys.getFirst().getTimestamp(), value);
            try {
                writer.addMutation(m);
            } catch (final MutationsRejectedException e) {
                LOGGER.error("Failed to create an accumulo key mutation");
                continue;
            }
            // If the GraphElement is an Edge then there will be 2 keys.
            if (keys.getSecond() != null) {
                final Mutation m2 = new Mutation(keys.getSecond().getRow());
                m2.put(keys.getSecond().getColumnFamily(), keys.getSecond().getColumnQualifier(), new ColumnVisibility(keys.getSecond().getColumnVisibility()), keys.getSecond().getTimestamp(), value);
                try {
                    writer.addMutation(m2);
                } catch (final MutationsRejectedException e) {
                    LOGGER.error("Failed to create an accumulo key mutation");
                }
            }
        }
    } else {
        throw new GafferRuntimeException("Could not find any elements to add to graph.", Status.BAD_REQUEST);
    }
    try {
        writer.close();
    } catch (final MutationsRejectedException e) {
        LOGGER.warn("Accumulo batch writer failed to close", e);
    }
}

Example 3 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class TestIngest method ingest.

public static void ingest(Connector connector, FileSystem fs, Opts opts, BatchWriterOpts bwOpts) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException, TableExistsException {
    long stopTime;
    byte[][] bytevals = generateValues(opts.dataSize);
    byte[] randomValue = new byte[opts.dataSize];
    Random random = new Random();
    long bytesWritten = 0;
    createTable(connector, opts);
    BatchWriter bw = null;
    FileSKVWriter writer = null;
    if (opts.outputFile != null) {
        Configuration conf = CachedConfiguration.getInstance();
        writer = FileOperations.getInstance().newWriterBuilder().forFile(opts.outputFile + "." + RFile.EXTENSION, fs, conf).withTableConfiguration(DefaultConfiguration.getInstance()).build();
        writer.startDefaultLocalityGroup();
    } else {
        bw = connector.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
        connector.securityOperations().changeUserAuthorizations(opts.getPrincipal(), AUTHS);
    }
    Text labBA = new Text(opts.columnVisibility.getExpression());
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < opts.rows; i++) {
        int rowid;
        if (opts.stride > 0) {
            rowid = ((i % opts.stride) * (opts.rows / opts.stride)) + (i / opts.stride);
        } else {
            rowid = i;
        }
        Text row = generateRow(rowid, opts.startRow);
        Mutation m = new Mutation(row);
        for (int j = 0; j < opts.cols; j++) {
            Text colf = new Text(opts.columnFamily);
            Text colq = new Text(FastFormat.toZeroPaddedString(j, 7, 10, COL_PREFIX));
            if (writer != null) {
                Key key = new Key(row, colf, colq, labBA);
                if (opts.timestamp >= 0) {
                    key.setTimestamp(opts.timestamp);
                } else {
                    key.setTimestamp(startTime);
                }
                if (opts.delete) {
                    key.setDeleted(true);
                } else {
                    key.setDeleted(false);
                }
                bytesWritten += key.getSize();
                if (opts.delete) {
                    writer.append(key, new Value(new byte[0]));
                } else {
                    byte[] value;
                    if (opts.random != null) {
                        value = genRandomValue(random, randomValue, opts.random, rowid + opts.startRow, j);
                    } else {
                        value = bytevals[j % bytevals.length];
                    }
                    Value v = new Value(value);
                    writer.append(key, v);
                    bytesWritten += v.getSize();
                }
            } else {
                Key key = new Key(row, colf, colq, labBA);
                bytesWritten += key.getSize();
                if (opts.delete) {
                    if (opts.timestamp >= 0)
                        m.putDelete(colf, colq, opts.columnVisibility, opts.timestamp);
                    else
                        m.putDelete(colf, colq, opts.columnVisibility);
                } else {
                    byte[] value;
                    if (opts.random != null) {
                        value = genRandomValue(random, randomValue, opts.random, rowid + opts.startRow, j);
                    } else {
                        value = bytevals[j % bytevals.length];
                    }
                    bytesWritten += value.length;
                    if (opts.timestamp >= 0) {
                        m.put(colf, colq, opts.columnVisibility, opts.timestamp, new Value(value, true));
                    } else {
                        m.put(colf, colq, opts.columnVisibility, new Value(value, true));
                    }
                }
            }
        }
        if (bw != null)
            bw.addMutation(m);
    }
    if (writer != null) {
        writer.close();
    } else if (bw != null) {
        try {
            bw.close();
        } catch (MutationsRejectedException e) {
            if (e.getSecurityErrorCodes().size() > 0) {
                for (Entry<TabletId, Set<SecurityErrorCode>> entry : e.getSecurityErrorCodes().entrySet()) {
                    System.err.println("ERROR : Not authorized to write to : " + entry.getKey() + " due to " + entry.getValue());
                }
            }
            if (e.getConstraintViolationSummaries().size() > 0) {
                for (ConstraintViolationSummary cvs : e.getConstraintViolationSummaries()) {
                    System.err.println("ERROR : Constraint violates : " + cvs);
                }
            }
            throw e;
        }
    }
    stopTime = System.currentTimeMillis();
    int totalValues = opts.rows * opts.cols;
    double elapsed = (stopTime - startTime) / 1000.0;
    System.out.printf("%,12d records written | %,8d records/sec | %,12d bytes written | %,8d bytes/sec | %6.3f secs   %n", totalValues, (int) (totalValues / elapsed), bytesWritten, (int) (bytesWritten / elapsed), elapsed);
}

Example 4 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class ConcurrencyIT method runTest.

static void runTest(Connector c, String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, MutationsRejectedException, Exception, InterruptedException {
    c.tableOperations().create(tableName);
    IteratorSetting is = new IteratorSetting(10, SlowIterator.class);
    SlowIterator.setSleepTime(is, 50);
    c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc, IteratorScope.majc));
    c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1.0");
    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
    for (int i = 0; i < 50; i++) {
        Mutation m = new Mutation(new Text(String.format("%06d", i)));
        m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
        bw.addMutation(m);
    }
    bw.flush();
    ScanTask st0 = new ScanTask(c, tableName, 300);
    st0.start();
    ScanTask st1 = new ScanTask(c, tableName, 100);
    st1.start();
    sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
    c.tableOperations().flush(tableName, null, null, true);
    for (int i = 0; i < 50; i++) {
        Mutation m = new Mutation(new Text(String.format("%06d", i)));
        m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
        bw.addMutation(m);
    }
    bw.flush();
    ScanTask st2 = new ScanTask(c, tableName, 100);
    st2.start();
    st1.join();
    st2.join();
    if (st1.count != 50)
        throw new Exception("Thread 1 did not see 50, saw " + st1.count);
    if (st2.count != 50)
        throw new Exception("Thread 2 did not see 50, saw " + st2.count);
    ScanTask st3 = new ScanTask(c, tableName, 150);
    st3.start();
    sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
    c.tableOperations().flush(tableName, null, null, false);
    st3.join();
    if (st3.count != 50)
        throw new Exception("Thread 3 did not see 50, saw " + st3.count);
    st0.join();
    if (st0.count != 50)
        throw new Exception("Thread 0 did not see 50, saw " + st0.count);
    bw.close();
}

Example 5 with MutationsRejectedException

use of org.apache.accumulo.core.client.MutationsRejectedException in project accumulo by apache.

the class BatchWriterFlushIT method runFlushTest.

private void runFlushTest(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, MutationsRejectedException, Exception {
    BatchWriter bw = getConnector().createBatchWriter(tableName, new BatchWriterConfig());
    try (Scanner scanner = getConnector().createScanner(tableName, Authorizations.EMPTY)) {
        Random r = new Random();
        for (int i = 0; i < 4; i++) {
            for (int j = 0; j < NUM_TO_FLUSH; j++) {
                int row = i * NUM_TO_FLUSH + j;
                Mutation m = new Mutation(new Text(String.format("r_%10d", row)));
                m.put(new Text("cf"), new Text("cq"), new Value(("" + row).getBytes()));
                bw.addMutation(m);
            }
            bw.flush();
            for (int k = 0; k < 10; k++) {
                int rowToLookup = r.nextInt(NUM_TO_FLUSH) + i * NUM_TO_FLUSH;
                scanner.setRange(new Range(new Text(String.format("r_%10d", rowToLookup))));
                Iterator<Entry<Key, Value>> iter = scanner.iterator();
                if (!iter.hasNext())
                    throw new Exception(" row " + rowToLookup + " not found after flush");
                Entry<Key, Value> entry = iter.next();
                if (iter.hasNext())
                    throw new Exception("Scanner returned too much");
                verifyEntry(rowToLookup, entry);
            }
            // scan all data just flushed
            scanner.setRange(new Range(new Text(String.format("r_%10d", i * NUM_TO_FLUSH)), true, new Text(String.format("r_%10d", (i + 1) * NUM_TO_FLUSH)), false));
            Iterator<Entry<Key, Value>> iter = scanner.iterator();
            for (int j = 0; j < NUM_TO_FLUSH; j++) {
                int row = i * NUM_TO_FLUSH + j;
                if (!iter.hasNext())
                    throw new Exception("Scan stopped permaturely at " + row);
                Entry<Key, Value> entry = iter.next();
                verifyEntry(row, entry);
            }
            if (iter.hasNext())
                throw new Exception("Scanner returned too much");
        }
        bw.close();
        // test adding a mutation to a closed batch writer
        boolean caught = false;
        try {
            bw.addMutation(new Mutation(new Text("foobar")));
        } catch (IllegalStateException ise) {
            caught = true;
        }
        if (!caught) {
            throw new Exception("Adding to closed batch writer did not fail");
        }
    }
}