Examples with CompactionConfig org.apache.accumulo.core.client.admin.CompactionConfig used on opensource projects

Example 21 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactCommand method execute.

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
    if (cl.hasOption(cancelOpt.getLongOpt())) {
        cancel = true;
        if (cl.getOptions().length > 2) {
            throw new IllegalArgumentException("Can not specify other options with cancel");
        }
    } else {
        cancel = false;
    }
    compactionConfig = new CompactionConfig();
    compactionConfig.setFlush(!cl.hasOption(noFlushOption.getOpt()));
    compactionConfig.setWait(cl.hasOption(waitOpt.getOpt()));
    compactionConfig.setStartRow(OptUtil.getStartRow(cl));
    compactionConfig.setEndRow(OptUtil.getEndRow(cl));
    if (cl.hasOption(profileOpt.getOpt())) {
        List<IteratorSetting> iterators = shellState.iteratorProfiles.get(cl.getOptionValue(profileOpt.getOpt()));
        if (iterators == null) {
            Shell.log.error("Profile {} does not exist", cl.getOptionValue(profileOpt.getOpt()));
            return -1;
        }
        compactionConfig.setIterators(new ArrayList<>(iterators));
    }
    setupConfigurableCompaction(cl, compactionConfig);
    if (cl.hasOption(strategyOpt.getOpt())) {
        if (cl.hasOption(selectorOpt.getLongOpt()) || cl.hasOption(configurerOpt.getLongOpt())) {
            throw new IllegalArgumentException("Can not specify a strategy with a selector or configurer");
        }
        configureCompactionStrat(cl);
    } else {
        if (cl.hasOption(selectorOpt.getLongOpt())) {
            compactionConfig.setSelector(new PluginConfig(cl.getOptionValue(selectorOpt.getLongOpt()), ShellUtil.parseMapOpt(cl, selectorConfigOpt)));
        }
        if (cl.hasOption(configurerOpt.getLongOpt())) {
            compactionConfig.setConfigurer(new PluginConfig(cl.getOptionValue(configurerOpt.getLongOpt()), ShellUtil.parseMapOpt(cl, configurerConfigOpt)));
        }
    }
    if (cl.hasOption(hintsOption.getLongOpt())) {
        compactionConfig.setExecutionHints(ShellUtil.parseMapOpt(cl, hintsOption));
    }
    return super.execute(fullCommand, cl, shellState);
}

Example 22 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class SummaryIT method compactionSelectorTest.

@Test
public void compactionSelectorTest() throws Exception {
    // Create a compaction config that will filter out foos if there are too many. Uses summary
    // data to know if there are too many foos.
    PluginConfig csc = new PluginConfig(FooSelector.class.getName());
    CompactionConfig compactConfig = new CompactionConfig().setSelector(csc);
    compactionTest(compactConfig);
}

Example 23 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class SummaryIT method tooLargeTest.

@Test
public void tooLargeTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        NewTableConfiguration ntc = new NewTableConfiguration();
        SummarizerConfiguration sc1 = SummarizerConfiguration.builder(BigSummarizer.class).build();
        ntc.enableSummarization(sc1);
        c.tableOperations().create(table, ntc);
        try (BatchWriter bw = c.createBatchWriter(table)) {
            write(bw, "a_large", "f1", "q1", "v1");
            write(bw, "v_small", "f1", "q1", "v2");
        }
        c.tableOperations().flush(table, null, null, true);
        Summary summary = c.tableOperations().summaries(table).retrieve().get(0);
        assertEquals(1, summary.getFileStatistics().getLarge());
        assertEquals(0, summary.getFileStatistics().getMissing());
        assertEquals(0, summary.getFileStatistics().getExtra());
        assertEquals(0, summary.getFileStatistics().getDeleted());
        assertEquals(1, summary.getFileStatistics().getInaccurate());
        assertEquals(1, summary.getFileStatistics().getTotal());
        assertEquals(Collections.emptyMap(), summary.getStatistics());
        // create situation where one tablet has summary data and one does not because the summary
        // data
        // was too large
        c.tableOperations().addSplits(table, new TreeSet<>(Collections.singleton(new Text("m"))));
        c.tableOperations().compact(table, new CompactionConfig().setWait(true));
        summary = c.tableOperations().summaries(table).retrieve().get(0);
        assertEquals(1, summary.getFileStatistics().getLarge());
        assertEquals(0, summary.getFileStatistics().getMissing());
        assertEquals(0, summary.getFileStatistics().getExtra());
        assertEquals(0, summary.getFileStatistics().getDeleted());
        assertEquals(1, summary.getFileStatistics().getInaccurate());
        assertEquals(2, summary.getFileStatistics().getTotal());
        HashMap<String, Long> expected = new HashMap<>();
        for (int i = 0; i < 10; i++) {
            expected.put(String.format("%09x", i), i * 19L);
        }
        assertEquals(expected, summary.getStatistics());
    }
}

Example 24 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class SummaryIT method compactionStrategyTest.

@SuppressWarnings("removal")
@Test
public void compactionStrategyTest() throws Exception {
    var csc = new org.apache.accumulo.core.client.admin.CompactionStrategyConfig(FooCS.class.getName());
    CompactionConfig compactConfig = new CompactionConfig().setCompactionStrategy(csc);
    compactionTest(compactConfig);
}

Example 25 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class SummaryIT method basicSummaryTest.

@Test
public void basicSummaryTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
        NewTableConfiguration ntc = new NewTableConfiguration();
        SummarizerConfiguration sc1 = SummarizerConfiguration.builder(BasicSummarizer.class.getName()).build();
        ntc.enableSummarization(sc1);
        c.tableOperations().create(table, ntc);
        BatchWriter bw = writeData(table, c);
        Collection<Summary> summaries = c.tableOperations().summaries(table).flush(false).retrieve();
        assertEquals(0, summaries.size());
        LongSummaryStatistics stats = getTimestampStats(table, c);
        summaries = c.tableOperations().summaries(table).flush(true).retrieve();
        checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        Mutation m = new Mutation(String.format("r%09x", 999));
        m.put("f1", "q1", "999-0");
        m.putDelete("f1", "q2");
        bw.addMutation(m);
        bw.flush();
        c.tableOperations().flush(table, null, null, true);
        stats = getTimestampStats(table, c);
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_002L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 1L);
        bw.close();
        c.tableOperations().compact(table, new CompactionConfig().setWait(true));
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        // split tablet into two
        String sp1 = String.format("r%09x", 50_000);
        addSplits(table, c, sp1);
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        // compact 2nd tablet
        c.tableOperations().compact(table, new CompactionConfig().setStartRow(new Text(sp1)).setWait(true));
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 2, 0, 1, TOTAL_STAT, 113_999L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        // get summaries for first tablet
        stats = getTimestampStats(table, c, sp1, null);
        summaries = c.tableOperations().summaries(table).startRow(sp1).retrieve();
        checkSummaries(summaries, sc1, 1, 0, 0, TOTAL_STAT, 49_999L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        // compact all tablets and regenerate all summaries
        c.tableOperations().compact(table, new CompactionConfig());
        summaries = c.tableOperations().summaries(table).retrieve();
        stats = getTimestampStats(table, c);
        checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
        summaries = c.tableOperations().summaries(table).startRow(String.format("r%09x", 75_000)).endRow(String.format("r%09x", 80_000)).retrieve();
        Summary summary = Iterables.getOnlyElement(summaries);
        assertEquals(1, summary.getFileStatistics().getTotal());
        assertEquals(1, summary.getFileStatistics().getExtra());
        long total = summary.getStatistics().get(TOTAL_STAT);
        assertTrue("Total " + total + " out of expected range", total > 0 && total <= 10_000);
        // test adding and removing
        c.tableOperations().removeSummarizers(table, sc -> sc.getClassName().contains("foo"));
        List<SummarizerConfiguration> summarizers = c.tableOperations().listSummarizers(table);
        assertEquals(1, summarizers.size());
        assertTrue(summarizers.contains(sc1));
        c.tableOperations().removeSummarizers(table, sc -> sc.getClassName().equals(BasicSummarizer.class.getName()));
        summarizers = c.tableOperations().listSummarizers(table);
        assertEquals(0, summarizers.size());
        c.tableOperations().compact(table, new CompactionConfig().setWait(true));
        summaries = c.tableOperations().summaries(table).retrieve();
        assertEquals(0, summaries.size());
        c.tableOperations().addSummarizers(table, sc1);
        c.tableOperations().compact(table, new CompactionConfig().setWait(true));
        summaries = c.tableOperations().summaries(table).retrieve();
        checkSummaries(summaries, sc1, 2, 0, 0, TOTAL_STAT, 100_000L, MIN_TIMESTAMP_STAT, stats.getMin(), MAX_TIMESTAMP_STAT, stats.getMax(), DELETES_STAT, 0L);
    }
}