Examples with CompactionStrategyConfig org.apache.accumulo.core.client.admin.CompactionStrategyConfig used on opensource projects

Example 6 with CompactionStrategyConfig

use of org.apache.accumulo.core.client.admin.CompactionStrategyConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testDropA.

@Test
public void testDropA() throws Exception {
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    writeFlush(c, tableName, "a");
    writeFlush(c, tableName, "b");
    // create a file that starts with A containing rows 'a' and 'b'
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    writeFlush(c, tableName, "c");
    writeFlush(c, tableName, "d");
    // drop files that start with A
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig(TestCompactionStrategy.class.getName());
    csConfig.setOptions(ImmutableMap.of("dropPrefix", "A", "inputPrefix", "F"));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(ImmutableSet.of("c", "d"), getRows(c, tableName));
    // this compaction should not drop files starting with A
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    Assert.assertEquals(ImmutableSet.of("c", "d"), getRows(c, tableName));
}

Example 7 with CompactionStrategyConfig

use of org.apache.accumulo.core.client.admin.CompactionStrategyConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testDropNone.

private void testDropNone(Map<String, String> options) throws Exception {
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    writeFlush(c, tableName, "a");
    writeFlush(c, tableName, "b");
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig(TestCompactionStrategy.class.getName());
    csConfig.setOptions(options);
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(ImmutableSet.of("a", "b"), getRows(c, tableName));
}

Example 8 with CompactionStrategyConfig

use of org.apache.accumulo.core.client.admin.CompactionStrategyConfig in project accumulo by apache.

the class SummaryIT method compactionTest.

@Test
public void compactionTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(FooCounter.class.getName()).build();
    ntc.enableSummarization(sc1);
    c.tableOperations().create(table, ntc);
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "bar1", "f1", "q1", "v1");
        write(bw, "bar2", "f1", "q1", "v2");
        write(bw, "foo1", "f1", "q1", "v3");
    }
    // Create a compaction config that will filter out foos if there are too many. Uses summary data to know if there are too many foos.
    CompactionStrategyConfig csc = new CompactionStrategyConfig(FooCS.class.getName());
    List<IteratorSetting> iterators = Collections.singletonList(new IteratorSetting(100, FooFilter.class));
    CompactionConfig compactConfig = new CompactionConfig().setFlush(true).setCompactionStrategy(csc).setIterators(iterators).setWait(true);
    // this compaction should make no changes because there are less foos than bars
    c.tableOperations().compact(table, compactConfig);
    try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
        Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
        Map<String, Long> counts = // convert to row
        stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
        r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
        groupingBy(identity(), counting()));
        Assert.assertEquals(1l, (long) counts.getOrDefault("foo", 0l));
        Assert.assertEquals(2l, (long) counts.getOrDefault("bar", 0l));
        Assert.assertEquals(2, counts.size());
    }
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "foo2", "f1", "q1", "v4");
        write(bw, "foo3", "f1", "q1", "v5");
        write(bw, "foo4", "f1", "q1", "v6");
    }
    // this compaction should remove all foos because there are more foos than bars
    c.tableOperations().compact(table, compactConfig);
    try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
        Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
        Map<String, Long> counts = // convert to row
        stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
        r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
        groupingBy(identity(), counting()));
        Assert.assertEquals(0l, (long) counts.getOrDefault("foo", 0l));
        Assert.assertEquals(2l, (long) counts.getOrDefault("bar", 0l));
        Assert.assertEquals(1, counts.size());
    }
}

Example 9 with CompactionStrategyConfig

use of org.apache.accumulo.core.client.admin.CompactionStrategyConfig in project accumulo by apache.

the class CompactionStrategyConfigUtil method decode.

public static CompactionStrategyConfig decode(DataInput din) throws IOException {
    if (din.readInt() != MAGIC) {
        throw new IllegalArgumentException("Unexpected MAGIC ");
    }
    if (din.readByte() != 1) {
        throw new IllegalArgumentException("Unexpected version");
    }
    String classname = din.readUTF();
    int numEntries = din.readInt();
    HashMap<String, String> opts = new HashMap<>();
    for (int i = 0; i < numEntries; i++) {
        String k = din.readUTF();
        String v = din.readUTF();
        opts.put(k, v);
    }
    return new CompactionStrategyConfig(classname).setOptions(opts);
}