Examples with BatchWriter org.apache.accumulo.core.client.BatchWriter used on opensource projects

Example 26 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class LargeSplitRowIT method userAddedSplit.

// User added split
@Test(timeout = 60 * 1000)
public void userAddedSplit() throws Exception {
    log.info("User added split");
    // make a table and lower the TABLE_END_ROW_MAX_SIZE property
    final String tableName = getUniqueNames(1)[0];
    final Connector conn = getConnector();
    conn.tableOperations().create(tableName);
    conn.tableOperations().setProperty(tableName, Property.TABLE_MAX_END_ROW_SIZE.getKey(), "1000");
    // Create a BatchWriter and add a mutation to the table
    BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
    Mutation m = new Mutation("Row");
    m.put("cf", "cq", "value");
    batchWriter.addMutation(m);
    batchWriter.close();
    // Create a split point that is too large to be an end row and fill it with all 'm'
    SortedSet<Text> partitionKeys = new TreeSet<>();
    byte[] data = new byte[(int) (ConfigurationTypeHelper.getFixedMemoryAsBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)];
    for (int i = 0; i < data.length; i++) {
        data[i] = 'm';
    }
    partitionKeys.add(new Text(data));
    // try to add the split point that is too large, if the split point is created the test fails.
    try {
        conn.tableOperations().addSplits(tableName, partitionKeys);
        Assert.fail();
    } catch (AccumuloServerException e) {
    }
    // Make sure that the information that was written to the table before we tried to add the split point is still correct
    int counter = 0;
    try (Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY)) {
        for (Entry<Key, Value> entry : scanner) {
            counter++;
            Key k = entry.getKey();
            Assert.assertEquals("Row", k.getRow().toString());
            Assert.assertEquals("cf", k.getColumnFamily().toString());
            Assert.assertEquals("cq", k.getColumnQualifier().toString());
            Assert.assertEquals("value", entry.getValue().toString());
        }
    }
    // Make sure there is only one line in the table
    Assert.assertEquals(1, counter);
}

Example 27 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class LargeSplitRowIT method automaticSplitLater.

@Test(timeout = 60 * 1000)
public void automaticSplitLater() throws Exception {
    log.info("Split later");
    automaticSplit(15, 1);
    final Connector conn = getConnector();
    String tableName = new String();
    java.util.Iterator<String> iterator = conn.tableOperations().list().iterator();
    while (iterator.hasNext()) {
        String curr = iterator.next();
        if (!curr.startsWith(Namespace.ACCUMULO + ".")) {
            tableName = curr;
        }
    }
    // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row
    BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
    byte[] data = new byte[10];
    // Fill key with all j's except for last spot which alternates through 1 through 10 for every j value
    for (int j = 15; j < 150; j += 1) {
        for (int i = 0; i < data.length - 1; i++) {
            data[i] = (byte) j;
        }
        for (int i = 0; i < 25; i++) {
            data[data.length - 1] = (byte) i;
            Mutation m = new Mutation(data);
            m.put("cf", "cq", "value");
            batchWriter.addMutation(m);
        }
    }
    // Flush the BatchWriter and table and sleep for a bit to make sure that there is enough time for the table to split if need be.
    batchWriter.close();
    conn.tableOperations().flush(tableName, new Text(), new Text("z"), true);
    // Make sure a split occurs
    while (conn.tableOperations().listSplits(tableName).size() == 0) {
        Thread.sleep(250);
    }
    Assert.assertTrue(0 < conn.tableOperations().listSplits(tableName).size());
}

Example 28 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class LargeSplitRowIT method automaticSplit.

private void automaticSplit(int max, int spacing) throws Exception {
    // make a table and lower the configure properties
    final String tableName = getUniqueNames(1)[0];
    final Connector conn = getConnector();
    conn.tableOperations().create(tableName);
    conn.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
    conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
    conn.tableOperations().setProperty(tableName, Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "64");
    conn.tableOperations().setProperty(tableName, Property.TABLE_MAX_END_ROW_SIZE.getKey(), "1000");
    // Create a BatchWriter and key for a table entry that is longer than the allowed size for an end row
    BatchWriter batchWriter = conn.createBatchWriter(tableName, new BatchWriterConfig());
    byte[] data = new byte[(int) (ConfigurationTypeHelper.getFixedMemoryAsBytes(Property.TABLE_MAX_END_ROW_SIZE.getDefaultValue()) + 2)];
    // Fill key with all j's except for last spot which alternates through 1 through 10 for every j value
    for (int j = 0; j < max; j += spacing) {
        for (int i = 0; i < data.length - 1; i++) {
            data[i] = (byte) j;
        }
        for (int i = 0; i < 10; i++) {
            data[data.length - 1] = (byte) i;
            Mutation m = new Mutation(data);
            m.put("cf", "cq", "value");
            batchWriter.addMutation(m);
        }
    }
    // Flush the BatchWriter and table and sleep for a bit to make sure that there is enough time for the table to split if need be.
    batchWriter.close();
    conn.tableOperations().flush(tableName, new Text(), new Text("z"), true);
    Thread.sleep(500);
    // Make sure all the data that was put in the table is still correct
    int count = 0;
    int extra = 10;
    try (Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY)) {
        for (Entry<Key, Value> entry : scanner) {
            if (extra == 10) {
                extra = 0;
                for (int i = 0; i < data.length - 1; i++) {
                    data[i] = (byte) count;
                }
                count += spacing;
            }
            Key k = entry.getKey();
            data[data.length - 1] = (byte) extra;
            String expected = new String(data, UTF_8);
            Assert.assertEquals(expected, k.getRow().toString());
            Assert.assertEquals("cf", k.getColumnFamily().toString());
            Assert.assertEquals("cq", k.getColumnQualifier().toString());
            Assert.assertEquals("value", entry.getValue().toString());
            extra++;
        }
    }
    Assert.assertEquals(10, extra);
    Assert.assertEquals(max, count);
    // Make sure no splits occured in the table
    Assert.assertEquals(0, conn.tableOperations().listSplits(tableName).size());
}

Example 29 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class BatchWriterIT method test.

@Test
public void test() throws Exception {
    // call the batchwriter with buffer of size zero
    String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    c.tableOperations().create(table);
    BatchWriterConfig config = new BatchWriterConfig();
    config.setMaxMemory(0);
    BatchWriter writer = c.createBatchWriter(table, config);
    Mutation m = new Mutation("row");
    m.put("cf", "cq", new Value("value".getBytes()));
    writer.addMutation(m);
    writer.close();
}

Example 30 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class BatchWriterInTabletServerIT method test.

private void test(String t1, String t2, Connector c, IteratorSetting itset, int numEntriesToWritePerEntry) throws Exception {
    // Write an entry to t1
    c.tableOperations().create(t1);
    Key k = new Key(new Text("row"), new Text("cf"), new Text("cq"));
    Value v = new Value("1".getBytes());
    {
        BatchWriterConfig config = new BatchWriterConfig();
        config.setMaxMemory(0);
        BatchWriter writer = c.createBatchWriter(t1, config);
        Mutation m = new Mutation(k.getRow());
        m.put(k.getColumnFamily(), k.getColumnQualifier(), v);
        writer.addMutation(m);
        writer.close();
    }
    // Create t2 with a combiner to count entries written to it
    c.tableOperations().create(t2);
    IteratorSetting summer = new IteratorSetting(2, "summer", SummingCombiner.class);
    LongCombiner.setEncodingType(summer, LongCombiner.Type.STRING);
    LongCombiner.setCombineAllColumns(summer, true);
    c.tableOperations().attachIterator(t2, summer);
    Map.Entry<Key, Value> actual;
    try (Scanner scanner = c.createScanner(t1, Authorizations.EMPTY)) {
        // Scan t1 with an iterator that writes to table t2
        scanner.addScanIterator(itset);
        actual = Iterators.getOnlyElement(scanner.iterator());
        Assert.assertTrue(actual.getKey().equals(k, PartialKey.ROW_COLFAM_COLQUAL));
        Assert.assertEquals(BatchWriterIterator.SUCCESS_VALUE, actual.getValue());
    }
    try (Scanner scanner = c.createScanner(t2, Authorizations.EMPTY)) {
        // ensure entries correctly wrote to table t2
        actual = Iterators.getOnlyElement(scanner.iterator());
        log.debug("t2 entry is " + actual.getKey().toStringNoTime() + " -> " + actual.getValue());
        Assert.assertTrue(actual.getKey().equals(k, PartialKey.ROW_COLFAM_COLQUAL));
        Assert.assertEquals(numEntriesToWritePerEntry, Integer.parseInt(actual.getValue().toString()));
    }
    c.tableOperations().delete(t1);
    c.tableOperations().delete(t2);
}