Examples with CompactionConfig org.apache.accumulo.core.client.admin.CompactionConfig used on opensource projects

Example 46 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testDropA.

@Test
public void testDropA() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        writeFlush(c, tableName, "a");
        writeFlush(c, tableName, "b");
        // create a file that starts with A containing rows 'a' and 'b'
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
        writeFlush(c, tableName, "c");
        writeFlush(c, tableName, "d");
        // drop files that start with A
        CompactionStrategyConfig csConfig = new CompactionStrategyConfig(TestCompactionStrategy.class.getName());
        csConfig.setOptions(Map.of("dropPrefix", "A", "inputPrefix", "F"));
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
        assertEquals(Set.of("c", "d"), getRows(c, tableName));
        // this compaction should not drop files starting with A
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
        assertEquals(Set.of("c", "d"), getRows(c, tableName));
    }
}

Example 47 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testIterators.

@Test
public void testIterators() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        writeFlush(c, tableName, "a");
        writeFlush(c, tableName, "b");
        // create a file that starts with A containing rows 'a' and 'b'
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
        writeFlush(c, tableName, "c");
        writeFlush(c, tableName, "d");
        assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName));
        // drop files that start with A
        CompactionStrategyConfig csConfig = new CompactionStrategyConfig(TestCompactionStrategy.class.getName());
        csConfig.setOptions(Map.of("inputPrefix", "F"));
        IteratorSetting iterConf = new IteratorSetting(21, "myregex", RegExFilter.class);
        RegExFilter.setRegexs(iterConf, "a|c", null, null, null, false);
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig).setIterators(Arrays.asList(iterConf)));
        // compaction strategy should only be applied to one file. If its applied to both, then row
        // 'b'
        // would be dropped by filter.
        assertEquals(Set.of("a", "b", "c"), getRows(c, tableName));
        assertEquals(2, FunctionalTestUtils.countRFiles(c, tableName));
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
        // ensure that iterator is not applied
        assertEquals(Set.of("a", "b", "c"), getRows(c, tableName));
        assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName));
    }
}

Example 48 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testFileSize.

@Test
public void testFileSize() throws Exception {
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        String tableName = getUniqueNames(1)[0];
        c.tableOperations().create(tableName);
        // write random data because its very unlikely it will compress
        writeRandomValue(c, tableName, 1 << 16);
        writeRandomValue(c, tableName, 1 << 16);
        writeRandomValue(c, tableName, 1 << 9);
        writeRandomValue(c, tableName, 1 << 7);
        writeRandomValue(c, tableName, 1 << 6);
        assertEquals(5, FunctionalTestUtils.countRFiles(c, tableName));
        CompactionStrategyConfig csConfig = new CompactionStrategyConfig(SizeCompactionStrategy.class.getName());
        csConfig.setOptions(Map.of("size", "" + (1 << 15)));
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
        assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName));
        csConfig = new CompactionStrategyConfig(SizeCompactionStrategy.class.getName());
        csConfig.setOptions(Map.of("size", "" + (1 << 17)));
        c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
        assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName));
    }
}

Example 49 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactionExecutorIT method testDispatchUser.

@Test
public void testDispatchUser() throws Exception {
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        createTable(client, "dut1", "cs3");
        createTable(client, "dut2", "cs3", "special", "cs4");
        addFiles(client, "dut1", 6);
        addFiles(client, "dut2", 33);
        assertEquals(6, getFiles(client, "dut1").size());
        assertEquals(33, getFiles(client, "dut2").size());
        client.tableOperations().compact("dut1", new CompactionConfig().setWait(false));
        // The hint should cause the compaction to dispatch to service cs4 which will produce a
        // different number of files.
        client.tableOperations().compact("dut2", new CompactionConfig().setWait(false).setExecutionHints(Map.of("compaction_type", "special")));
        while (getFiles(client, "dut1").size() > 2 || getFiles(client, "dut2").size() > 3) {
            Thread.sleep(100);
        }
        assertEquals(2, getFiles(client, "dut1").size());
        assertEquals(3, getFiles(client, "dut2").size());
        // The way the compaction services were configured, they would never converge to one file for
        // the user compactions. However Accumulo will keep asking the planner for a plan until a user
        // compaction converges to one file. So cancel the compactions.
        client.tableOperations().cancelCompaction("dut1");
        client.tableOperations().cancelCompaction("dut2");
        assertEquals(2, getFiles(client, "dut1").size());
        assertEquals(3, getFiles(client, "dut2").size());
        client.tableOperations().compact("dut1", new CompactionConfig().setWait(true).setExecutionHints(Map.of("compact_all", "true")));
        client.tableOperations().compact("dut2", new CompactionConfig().setWait(true).setExecutionHints(Map.of("compact_all", "true")));
        assertEquals(1, getFiles(client, "dut1").size());
        assertEquals(1, getFiles(client, "dut2").size());
    }
}

Example 50 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactionExecutorIT method testConfigurer.

@Test
public void testConfigurer() throws Exception {
    String tableName = "tcc";
    try (AccumuloClient client = Accumulo.newClient().from(getClientProps()).build()) {
        var ntc = new NewTableConfiguration().setProperties(Map.of(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none"));
        client.tableOperations().create(tableName, ntc);
        byte[] data = new byte[100000];
        Arrays.fill(data, (byte) 65);
        try (var writer = client.createBatchWriter(tableName)) {
            for (int row = 0; row < 10; row++) {
                Mutation m = new Mutation(row + "");
                m.at().family("big").qualifier("stuff").put(data);
                writer.addMutation(m);
            }
        }
        client.tableOperations().flush(tableName, null, null, true);
        // without compression, expect file to be large
        long sizes = getFileSizes(client, tableName);
        assertTrue("Unexpected files sizes : " + sizes, sizes > data.length * 10 && sizes < data.length * 11);
        client.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setConfigurer(new PluginConfig(CompressionConfigurer.class.getName(), Map.of(CompressionConfigurer.LARGE_FILE_COMPRESSION_TYPE, "gz", CompressionConfigurer.LARGE_FILE_COMPRESSION_THRESHOLD, data.length + ""))));
        // after compacting with compression, expect small file
        sizes = getFileSizes(client, tableName);
        assertTrue("Unexpected files sizes : " + sizes, sizes < data.length);
        client.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
        // after compacting without compression, expect big files again
        sizes = getFileSizes(client, tableName);
        assertTrue("Unexpected files sizes : " + sizes, sizes > data.length * 10 && sizes < data.length * 11);
    }
}