Search in sources :

Example 11 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class CombinerIT method aggregationTest.

@Test
public void aggregationTest() throws Exception {
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class);
    SummingCombiner.setEncodingType(setting, Type.STRING);
    SummingCombiner.setColumns(setting, Collections.singletonList(new IteratorSetting.Column("cf")));
    c.tableOperations().attachIterator(tableName, setting);
    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
    for (int i = 0; i < 10; i++) {
        Mutation m = new Mutation("row1");
        m.put("cf".getBytes(), "col1".getBytes(), ("" + i).getBytes());
        bw.addMutation(m);
    }
    bw.close();
    checkSum(tableName, c);
}
Also used : Connector(org.apache.accumulo.core.client.Connector) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig) BatchWriter(org.apache.accumulo.core.client.BatchWriter) Mutation(org.apache.accumulo.core.data.Mutation) Test(org.junit.Test)

Example 12 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class ConcurrencyIT method runTest.

static void runTest(Connector c, String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, MutationsRejectedException, Exception, InterruptedException {
    c.tableOperations().create(tableName);
    IteratorSetting is = new IteratorSetting(10, SlowIterator.class);
    SlowIterator.setSleepTime(is, 50);
    c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.minc, IteratorScope.majc));
    c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1.0");
    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
    for (int i = 0; i < 50; i++) {
        Mutation m = new Mutation(new Text(String.format("%06d", i)));
        m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
        bw.addMutation(m);
    }
    bw.flush();
    ScanTask st0 = new ScanTask(c, tableName, 300);
    st0.start();
    ScanTask st1 = new ScanTask(c, tableName, 100);
    st1.start();
    sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
    c.tableOperations().flush(tableName, null, null, true);
    for (int i = 0; i < 50; i++) {
        Mutation m = new Mutation(new Text(String.format("%06d", i)));
        m.put(new Text("cf1"), new Text("cq1"), new Value("foo".getBytes(UTF_8)));
        bw.addMutation(m);
    }
    bw.flush();
    ScanTask st2 = new ScanTask(c, tableName, 100);
    st2.start();
    st1.join();
    st2.join();
    if (st1.count != 50)
        throw new Exception("Thread 1 did not see 50, saw " + st1.count);
    if (st2.count != 50)
        throw new Exception("Thread 2 did not see 50, saw " + st2.count);
    ScanTask st3 = new ScanTask(c, tableName, 150);
    st3.start();
    sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
    c.tableOperations().flush(tableName, null, null, false);
    st3.join();
    if (st3.count != 50)
        throw new Exception("Thread 3 did not see 50, saw " + st3.count);
    st0.join();
    if (st0.count != 50)
        throw new Exception("Thread 0 did not see 50, saw " + st0.count);
    bw.close();
}
Also used : IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) Value(org.apache.accumulo.core.data.Value) BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig) Text(org.apache.hadoop.io.Text) BatchWriter(org.apache.accumulo.core.client.BatchWriter) Mutation(org.apache.accumulo.core.data.Mutation) TableExistsException(org.apache.accumulo.core.client.TableExistsException) TableNotFoundException(org.apache.accumulo.core.client.TableNotFoundException) AccumuloSecurityException(org.apache.accumulo.core.client.AccumuloSecurityException) MutationsRejectedException(org.apache.accumulo.core.client.MutationsRejectedException) AccumuloException(org.apache.accumulo.core.client.AccumuloException)

Example 13 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class ConfigurableCompactionIT method writeFlush.

private void writeFlush(Connector conn, String tablename, String row) throws Exception {
    BatchWriter bw = conn.createBatchWriter(tablename, new BatchWriterConfig());
    Mutation m = new Mutation(row);
    m.put("", "", "");
    bw.addMutation(m);
    bw.close();
    conn.tableOperations().flush(tablename, null, null, true);
}
Also used : BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig) BatchWriter(org.apache.accumulo.core.client.BatchWriter) Mutation(org.apache.accumulo.core.data.Mutation)

Example 14 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class VolumeChooserIT method writeDataToTable.

public static void writeDataToTable(Connector connector, String tableName) throws Exception {
    // Write some data to the table
    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
    for (String s : rows) {
        Mutation m = new Mutation(new Text(s));
        m.put(EMPTY, EMPTY, EMPTY_VALUE);
        bw.addMutation(m);
    }
    bw.close();
}
Also used : BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig) Text(org.apache.hadoop.io.Text) BatchWriter(org.apache.accumulo.core.client.BatchWriter) Mutation(org.apache.accumulo.core.data.Mutation)

Example 15 with BatchWriter

use of org.apache.accumulo.core.client.BatchWriter in project accumulo by apache.

the class VolumeIT method testRelativePaths.

@Test
public void testRelativePaths() throws Exception {
    List<String> expected = new ArrayList<>();
    Connector connector = getConnector();
    String tableName = getUniqueNames(1)[0];
    connector.tableOperations().create(tableName, new NewTableConfiguration().withoutDefaultIterators());
    Table.ID tableId = Table.ID.of(connector.tableOperations().tableIdMap().get(tableName));
    SortedSet<Text> partitions = new TreeSet<>();
    // with some splits
    for (String s : "c,g,k,p,s,v".split(",")) partitions.add(new Text(s));
    connector.tableOperations().addSplits(tableName, partitions);
    BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
    // create two files in each tablet
    String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
    for (String s : rows) {
        Mutation m = new Mutation(s);
        m.put("cf1", "cq1", "1");
        bw.addMutation(m);
        expected.add(s + ":cf1:cq1:1");
    }
    bw.flush();
    connector.tableOperations().flush(tableName, null, null, true);
    for (String s : rows) {
        Mutation m = new Mutation(s);
        m.put("cf1", "cq1", "2");
        bw.addMutation(m);
        expected.add(s + ":cf1:cq1:2");
    }
    bw.close();
    connector.tableOperations().flush(tableName, null, null, true);
    verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
    connector.tableOperations().offline(tableName, true);
    connector.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
    try (Scanner metaScanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
        metaScanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
        BatchWriter mbw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
        for (Entry<Key, Value> entry : metaScanner) {
            String cq = entry.getKey().getColumnQualifier().toString();
            if (cq.startsWith(v1.toString())) {
                Path path = new Path(cq);
                String relPath = "/" + path.getParent().getName() + "/" + path.getName();
                Mutation fileMut = new Mutation(entry.getKey().getRow());
                fileMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
                fileMut.put(entry.getKey().getColumnFamily().toString(), relPath, entry.getValue().toString());
                mbw.addMutation(fileMut);
            }
        }
        mbw.close();
        connector.tableOperations().online(tableName, true);
        verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
        connector.tableOperations().compact(tableName, null, null, true, true);
        verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
        for (Entry<Key, Value> entry : metaScanner) {
            String cq = entry.getKey().getColumnQualifier().toString();
            Path path = new Path(cq);
            Assert.assertTrue("relative path not deleted " + path.toString(), path.depth() > 2);
        }
    }
}
Also used : Path(org.apache.hadoop.fs.Path) Connector(org.apache.accumulo.core.client.Connector) Scanner(org.apache.accumulo.core.client.Scanner) MetadataTable(org.apache.accumulo.core.metadata.MetadataTable) RootTable(org.apache.accumulo.core.metadata.RootTable) Table(org.apache.accumulo.core.client.impl.Table) ArrayList(java.util.ArrayList) Text(org.apache.hadoop.io.Text) KeyExtent(org.apache.accumulo.core.data.impl.KeyExtent) NewTableConfiguration(org.apache.accumulo.core.client.admin.NewTableConfiguration) TreeSet(java.util.TreeSet) Value(org.apache.accumulo.core.data.Value) BatchWriterConfig(org.apache.accumulo.core.client.BatchWriterConfig) BatchWriter(org.apache.accumulo.core.client.BatchWriter) Mutation(org.apache.accumulo.core.data.Mutation) Key(org.apache.accumulo.core.data.Key) Test(org.junit.Test)

Aggregations

BatchWriter (org.apache.accumulo.core.client.BatchWriter)402 Mutation (org.apache.accumulo.core.data.Mutation)360 Test (org.junit.Test)264 Value (org.apache.accumulo.core.data.Value)250 BatchWriterConfig (org.apache.accumulo.core.client.BatchWriterConfig)246 Text (org.apache.hadoop.io.Text)194 Key (org.apache.accumulo.core.data.Key)179 Scanner (org.apache.accumulo.core.client.Scanner)174 Connector (org.apache.accumulo.core.client.Connector)169 IteratorSetting (org.apache.accumulo.core.client.IteratorSetting)81 Authorizations (org.apache.accumulo.core.security.Authorizations)68 Range (org.apache.accumulo.core.data.Range)61 Entry (java.util.Map.Entry)51 Map (java.util.Map)50 BatchScanner (org.apache.accumulo.core.client.BatchScanner)46 MutationsRejectedException (org.apache.accumulo.core.client.MutationsRejectedException)44 TableNotFoundException (org.apache.accumulo.core.client.TableNotFoundException)40 HashMap (java.util.HashMap)38 ArrayList (java.util.ArrayList)36 Status (org.apache.accumulo.server.replication.proto.Replication.Status)32