Examples with CompactionConfig org.apache.accumulo.core.client.admin.CompactionConfig used on opensource projects

Example 1 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testConcurrent.

@Test
public void testConcurrent() throws Exception {
    // two compactions without iterators or strategy should be able to run concurrently
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    // write random data because its very unlikely it will compress
    writeRandomValue(c, tableName, 1 << 16);
    writeRandomValue(c, tableName, 1 << 16);
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(false));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName));
    writeRandomValue(c, tableName, 1 << 16);
    IteratorSetting iterConfig = new IteratorSetting(30, SlowIterator.class);
    SlowIterator.setSleepTime(iterConfig, 1000);
    long t1 = System.currentTimeMillis();
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(false).setIterators(Arrays.asList(iterConfig)));
    try {
        // this compaction should fail because previous one set iterators
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
        if (System.currentTimeMillis() - t1 < 2000)
            Assert.fail("Expected compaction to fail because another concurrent compaction set iterators");
    } catch (AccumuloException e) {
    }
}

Example 2 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testPerTableClasspath.

@Test
public void testPerTableClasspath() throws Exception {
    // Can't assume that a test-resource will be on the server's classpath
    Assume.assumeTrue(ClusterType.MINI == getClusterType());
    // test per-table classpath + user specified compaction strategy
    final Connector c = getConnector();
    final String tableName = getUniqueNames(1)[0];
    File target = new File(System.getProperty("user.dir"), "target");
    Assert.assertTrue(target.mkdirs() || target.isDirectory());
    File destFile = installJar(target, "/TestCompactionStrat.jar");
    c.tableOperations().create(tableName);
    c.instanceOperations().setProperty(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "context1", destFile.toString());
    c.tableOperations().setProperty(tableName, Property.TABLE_CLASSPATH.getKey(), "context1");
    c.tableOperations().addSplits(tableName, new TreeSet<>(Arrays.asList(new Text("efg"))));
    writeFlush(c, tableName, "a");
    writeFlush(c, tableName, "b");
    writeFlush(c, tableName, "h");
    writeFlush(c, tableName, "i");
    Assert.assertEquals(4, FunctionalTestUtils.countRFiles(c, tableName));
    // EfgCompactionStrat will only compact a tablet w/ end row of 'efg'. No other tablets are compacted.
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig("org.apache.accumulo.test.EfgCompactionStrat");
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    Assert.assertEquals(2, FunctionalTestUtils.countRFiles(c, tableName));
}

Example 3 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testIterators.

@Test
public void testIterators() throws Exception {
    // test compaction strategy + iterators
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    writeFlush(c, tableName, "a");
    writeFlush(c, tableName, "b");
    // create a file that starts with A containing rows 'a' and 'b'
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    writeFlush(c, tableName, "c");
    writeFlush(c, tableName, "d");
    Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName));
    // drop files that start with A
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig(TestCompactionStrategy.class.getName());
    csConfig.setOptions(ImmutableMap.of("inputPrefix", "F"));
    IteratorSetting iterConf = new IteratorSetting(21, "myregex", RegExFilter.class);
    RegExFilter.setRegexs(iterConf, "a|c", null, null, null, false);
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig).setIterators(Arrays.asList(iterConf)));
    // compaction strategy should only be applied to one file. If its applied to both, then row 'b' would be dropped by filter.
    Assert.assertEquals(ImmutableSet.of("a", "b", "c"), getRows(c, tableName));
    Assert.assertEquals(2, FunctionalTestUtils.countRFiles(c, tableName));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    // ensure that iterator is not applied
    Assert.assertEquals(ImmutableSet.of("a", "b", "c"), getRows(c, tableName));
    Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName));
}

Example 4 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testFileSize.

@Test
public void testFileSize() throws Exception {
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    // write random data because its very unlikely it will compress
    writeRandomValue(c, tableName, 1 << 16);
    writeRandomValue(c, tableName, 1 << 16);
    writeRandomValue(c, tableName, 1 << 9);
    writeRandomValue(c, tableName, 1 << 7);
    writeRandomValue(c, tableName, 1 << 6);
    Assert.assertEquals(5, FunctionalTestUtils.countRFiles(c, tableName));
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig(SizeCompactionStrategy.class.getName());
    csConfig.setOptions(ImmutableMap.of("size", "" + (1 << 15)));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName));
    csConfig = new CompactionStrategyConfig(SizeCompactionStrategy.class.getName());
    csConfig.setOptions(ImmutableMap.of("size", "" + (1 << 17)));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName));
}

Example 5 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class ProxyServer method compactTable.

@Override
public void compactTable(ByteBuffer login, String tableName, ByteBuffer startRow, ByteBuffer endRow, List<org.apache.accumulo.proxy.thrift.IteratorSetting> iterators, boolean flush, boolean wait, CompactionStrategyConfig compactionStrategy) throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, org.apache.accumulo.proxy.thrift.TableNotFoundException, org.apache.accumulo.proxy.thrift.AccumuloException, TException {
    try {
        CompactionConfig compactionConfig = new CompactionConfig().setStartRow(ByteBufferUtil.toText(startRow)).setEndRow(ByteBufferUtil.toText(endRow)).setIterators(getIteratorSettings(iterators)).setFlush(flush).setWait(wait);
        if (compactionStrategy != null) {
            org.apache.accumulo.core.client.admin.CompactionStrategyConfig ccc = new org.apache.accumulo.core.client.admin.CompactionStrategyConfig(compactionStrategy.getClassName());
            if (compactionStrategy.options != null)
                ccc.setOptions(compactionStrategy.options);
            compactionConfig.setCompactionStrategy(ccc);
        }
        getConnector(login).tableOperations().compact(tableName, compactionConfig);
    } catch (Exception e) {
        handleExceptionTNF(e);
    }
}