Examples with CompactionStrategyConfig org.apache.accumulo.core.client.admin.CompactionStrategyConfig used on opensource projects

Example 1 with CompactionStrategyConfig

use of org.apache.accumulo.core.client.admin.CompactionStrategyConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testPerTableClasspath.

@Test
public void testPerTableClasspath() throws Exception {
    // Can't assume that a test-resource will be on the server's classpath
    Assume.assumeTrue(ClusterType.MINI == getClusterType());
    // test per-table classpath + user specified compaction strategy
    final Connector c = getConnector();
    final String tableName = getUniqueNames(1)[0];
    File target = new File(System.getProperty("user.dir"), "target");
    Assert.assertTrue(target.mkdirs() || target.isDirectory());
    File destFile = installJar(target, "/TestCompactionStrat.jar");
    c.tableOperations().create(tableName);
    c.instanceOperations().setProperty(Property.VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "context1", destFile.toString());
    c.tableOperations().setProperty(tableName, Property.TABLE_CLASSPATH.getKey(), "context1");
    c.tableOperations().addSplits(tableName, new TreeSet<>(Arrays.asList(new Text("efg"))));
    writeFlush(c, tableName, "a");
    writeFlush(c, tableName, "b");
    writeFlush(c, tableName, "h");
    writeFlush(c, tableName, "i");
    Assert.assertEquals(4, FunctionalTestUtils.countRFiles(c, tableName));
    // EfgCompactionStrat will only compact a tablet w/ end row of 'efg'. No other tablets are compacted.
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig("org.apache.accumulo.test.EfgCompactionStrat");
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    Assert.assertEquals(2, FunctionalTestUtils.countRFiles(c, tableName));
}

Example 2 with CompactionStrategyConfig

use of org.apache.accumulo.core.client.admin.CompactionStrategyConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testIterators.

@Test
public void testIterators() throws Exception {
    // test compaction strategy + iterators
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    writeFlush(c, tableName, "a");
    writeFlush(c, tableName, "b");
    // create a file that starts with A containing rows 'a' and 'b'
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    writeFlush(c, tableName, "c");
    writeFlush(c, tableName, "d");
    Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName));
    // drop files that start with A
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig(TestCompactionStrategy.class.getName());
    csConfig.setOptions(ImmutableMap.of("inputPrefix", "F"));
    IteratorSetting iterConf = new IteratorSetting(21, "myregex", RegExFilter.class);
    RegExFilter.setRegexs(iterConf, "a|c", null, null, null, false);
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig).setIterators(Arrays.asList(iterConf)));
    // compaction strategy should only be applied to one file. If its applied to both, then row 'b' would be dropped by filter.
    Assert.assertEquals(ImmutableSet.of("a", "b", "c"), getRows(c, tableName));
    Assert.assertEquals(2, FunctionalTestUtils.countRFiles(c, tableName));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    // ensure that iterator is not applied
    Assert.assertEquals(ImmutableSet.of("a", "b", "c"), getRows(c, tableName));
    Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName));
}

Example 3 with CompactionStrategyConfig

use of org.apache.accumulo.core.client.admin.CompactionStrategyConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testFileSize.

@Test
public void testFileSize() throws Exception {
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    // write random data because its very unlikely it will compress
    writeRandomValue(c, tableName, 1 << 16);
    writeRandomValue(c, tableName, 1 << 16);
    writeRandomValue(c, tableName, 1 << 9);
    writeRandomValue(c, tableName, 1 << 7);
    writeRandomValue(c, tableName, 1 << 6);
    Assert.assertEquals(5, FunctionalTestUtils.countRFiles(c, tableName));
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig(SizeCompactionStrategy.class.getName());
    csConfig.setOptions(ImmutableMap.of("size", "" + (1 << 15)));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(3, FunctionalTestUtils.countRFiles(c, tableName));
    csConfig = new CompactionStrategyConfig(SizeCompactionStrategy.class.getName());
    csConfig.setOptions(ImmutableMap.of("size", "" + (1 << 17)));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName));
}

Example 4 with CompactionStrategyConfig

use of org.apache.accumulo.core.client.admin.CompactionStrategyConfig in project accumulo by apache.

the class CompactCommand method execute.

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
    if (cl.hasOption(cancelOpt.getLongOpt())) {
        cancel = true;
        if (cl.getOptions().length > 2) {
            throw new IllegalArgumentException("Can not specify other options with cancel");
        }
    } else {
        cancel = false;
    }
    compactionConfig = new CompactionConfig();
    compactionConfig.setFlush(!cl.hasOption(noFlushOption.getOpt()));
    compactionConfig.setWait(cl.hasOption(waitOpt.getOpt()));
    compactionConfig.setStartRow(OptUtil.getStartRow(cl));
    compactionConfig.setEndRow(OptUtil.getEndRow(cl));
    if (cl.hasOption(profileOpt.getOpt())) {
        List<IteratorSetting> iterators = shellState.iteratorProfiles.get(cl.getOptionValue(profileOpt.getOpt()));
        if (iterators == null) {
            Shell.log.error("Profile " + cl.getOptionValue(profileOpt.getOpt()) + " does not exist");
            return -1;
        }
        compactionConfig.setIterators(new ArrayList<>(iterators));
    }
    Map<String, String> configurableCompactOpt = getConfigurableCompactionStrategyOpts(cl);
    if (cl.hasOption(strategyOpt.getOpt())) {
        if (configurableCompactOpt.size() > 0)
            throw new IllegalArgumentException("Can not specify compaction strategy with file selection and file output options.");
        CompactionStrategyConfig csc = new CompactionStrategyConfig(cl.getOptionValue(strategyOpt.getOpt()));
        if (cl.hasOption(strategyConfigOpt.getOpt())) {
            Map<String, String> props = new HashMap<>();
            String[] keyVals = cl.getOptionValue(strategyConfigOpt.getOpt()).split(",");
            for (String keyVal : keyVals) {
                String[] sa = keyVal.split("=");
                props.put(sa[0], sa[1]);
            }
            csc.setOptions(props);
        }
        compactionConfig.setCompactionStrategy(csc);
    }
    if (configurableCompactOpt.size() > 0) {
        CompactionStrategyConfig csc = new CompactionStrategyConfig("org.apache.accumulo.tserver.compaction.strategies.ConfigurableCompactionStrategy");
        csc.setOptions(configurableCompactOpt);
        compactionConfig.setCompactionStrategy(csc);
    }
    return super.execute(fullCommand, cl, shellState);
}

Example 5 with CompactionStrategyConfig

use of org.apache.accumulo.core.client.admin.CompactionStrategyConfig in project accumulo by apache.

the class Tablet method compactAll.

public void compactAll(long compactionId, UserCompactionConfig compactionConfig) {
    boolean updateMetadata = false;
    synchronized (this) {
        if (lastCompactID >= compactionId)
            return;
        if (isMinorCompactionRunning()) {
            // want to wait for running minc to finish before starting majc, see ACCUMULO-3041
            if (compactionWaitInfo.compactionID == compactionId) {
                if (lastFlushID == compactionWaitInfo.flushID)
                    return;
            } else {
                compactionWaitInfo.compactionID = compactionId;
                compactionWaitInfo.flushID = lastFlushID;
                return;
            }
        }
        if (isClosing() || isClosed() || majorCompactionQueued.contains(MajorCompactionReason.USER) || isMajorCompactionRunning())
            return;
        CompactionStrategyConfig strategyConfig = compactionConfig.getCompactionStrategy();
        CompactionStrategy strategy = createCompactionStrategy(strategyConfig);
        MajorCompactionRequest request = new MajorCompactionRequest(extent, MajorCompactionReason.USER, tableConfiguration);
        request.setFiles(getDatafileManager().getDatafileSizes());
        try {
            if (strategy.shouldCompact(request)) {
                initiateMajorCompaction(MajorCompactionReason.USER);
            } else {
                majorCompactionState = CompactionState.IN_PROGRESS;
                updateMetadata = true;
                lastCompactID = compactionId;
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    if (updateMetadata) {
        try {
            // if multiple threads were allowed to update this outside of a sync block, then it would be
            // a race condition
            MetadataTableUtil.updateTabletCompactID(extent, compactionId, getTabletServer(), getTabletServer().getLock());
        } finally {
            synchronized (this) {
                majorCompactionState = null;
                this.notifyAll();
            }
        }
    }
}