Examples with CompactionConfig org.apache.accumulo.core.client.admin.CompactionConfig used on opensource projects

Example 11 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class CompactCommand method execute.

@Override
public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws Exception {
    if (cl.hasOption(cancelOpt.getLongOpt())) {
        cancel = true;
        if (cl.getOptions().length > 2) {
            throw new IllegalArgumentException("Can not specify other options with cancel");
        }
    } else {
        cancel = false;
    }
    compactionConfig = new CompactionConfig();
    compactionConfig.setFlush(!cl.hasOption(noFlushOption.getOpt()));
    compactionConfig.setWait(cl.hasOption(waitOpt.getOpt()));
    compactionConfig.setStartRow(OptUtil.getStartRow(cl));
    compactionConfig.setEndRow(OptUtil.getEndRow(cl));
    if (cl.hasOption(profileOpt.getOpt())) {
        List<IteratorSetting> iterators = shellState.iteratorProfiles.get(cl.getOptionValue(profileOpt.getOpt()));
        if (iterators == null) {
            Shell.log.error("Profile " + cl.getOptionValue(profileOpt.getOpt()) + " does not exist");
            return -1;
        }
        compactionConfig.setIterators(new ArrayList<>(iterators));
    }
    Map<String, String> configurableCompactOpt = getConfigurableCompactionStrategyOpts(cl);
    if (cl.hasOption(strategyOpt.getOpt())) {
        if (configurableCompactOpt.size() > 0)
            throw new IllegalArgumentException("Can not specify compaction strategy with file selection and file output options.");
        CompactionStrategyConfig csc = new CompactionStrategyConfig(cl.getOptionValue(strategyOpt.getOpt()));
        if (cl.hasOption(strategyConfigOpt.getOpt())) {
            Map<String, String> props = new HashMap<>();
            String[] keyVals = cl.getOptionValue(strategyConfigOpt.getOpt()).split(",");
            for (String keyVal : keyVals) {
                String[] sa = keyVal.split("=");
                props.put(sa[0], sa[1]);
            }
            csc.setOptions(props);
        }
        compactionConfig.setCompactionStrategy(csc);
    }
    if (configurableCompactOpt.size() > 0) {
        CompactionStrategyConfig csc = new CompactionStrategyConfig("org.apache.accumulo.tserver.compaction.strategies.ConfigurableCompactionStrategy");
        csc.setOptions(configurableCompactOpt);
        compactionConfig.setCompactionStrategy(csc);
    }
    return super.execute(fullCommand, cl, shellState);
}

Example 12 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class SampleIT method testSampleNotPresent.

@Test
public void testSampleNotPresent() throws Exception {
    Connector conn = getConnector();
    String tableName = getUniqueNames(1)[0];
    String clone = tableName + "_clone";
    conn.tableOperations().create(tableName);
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    TreeMap<Key, Value> expected = new TreeMap<>();
    writeData(bw, SC1, expected);
    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
    Scanner isoScanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
    isoScanner.setBatchSize(10);
    Scanner csiScanner = new ClientSideIteratorScanner(conn.createScanner(tableName, Authorizations.EMPTY));
    BatchScanner bScanner = conn.createBatchScanner(tableName, Authorizations.EMPTY, 2);
    bScanner.setRanges(Arrays.asList(new Range()));
    // ensure sample not present exception occurs when sampling is not configured
    assertSampleNotPresent(SC1, scanner, isoScanner, bScanner, csiScanner);
    conn.tableOperations().flush(tableName, null, null, true);
    Scanner oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    assertSampleNotPresent(SC1, scanner, isoScanner, bScanner, csiScanner, oScanner);
    // configure sampling, however there exist an rfile w/o sample data... so should still see sample not present exception
    updateSamplingConfig(conn, tableName, SC1);
    // create clone with new config
    oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    assertSampleNotPresent(SC1, scanner, isoScanner, bScanner, csiScanner, oScanner);
    // create rfile with sample data present
    conn.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    // should be able to scan sample now
    oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    setSamplerConfig(SC1, scanner, csiScanner, isoScanner, bScanner, oScanner);
    check(expected, scanner, isoScanner, bScanner, csiScanner, oScanner);
    // change sampling config
    updateSamplingConfig(conn, tableName, SC2);
    // create clone with new config
    oScanner = newOfflineScanner(conn, tableName, clone, SC2);
    // rfile should have different sample config than table, and scan should not work
    assertSampleNotPresent(SC2, scanner, isoScanner, bScanner, csiScanner, oScanner);
    // create rfile that has same sample data as table config
    conn.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    // should be able to scan sample now
    updateExpected(SC2, expected);
    oScanner = newOfflineScanner(conn, tableName, clone, SC2);
    setSamplerConfig(SC2, scanner, csiScanner, isoScanner, bScanner, oScanner);
    check(expected, scanner, isoScanner, bScanner, csiScanner, oScanner);
}

Example 13 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo-examples by apache.

the class SampleExample method main.

public static void main(String[] args) throws Exception {
    Opts opts = new Opts();
    BatchWriterOpts bwOpts = new BatchWriterOpts();
    opts.parseArgs(RandomBatchWriter.class.getName(), args, bwOpts);
    Connector conn = opts.getConnector();
    if (!conn.tableOperations().exists(opts.getTableName())) {
        conn.tableOperations().create(opts.getTableName());
    } else {
        System.out.println("Table exists, not doing anything.");
        return;
    }
    // write some data
    BatchWriter bw = conn.createBatchWriter(opts.getTableName(), bwOpts.getBatchWriterConfig());
    bw.addMutation(createMutation("9225", "abcde", "file://foo.txt"));
    bw.addMutation(createMutation("8934", "accumulo scales", "file://accumulo_notes.txt"));
    bw.addMutation(createMutation("2317", "milk, eggs, bread, parmigiano-reggiano", "file://groceries/9/txt"));
    bw.addMutation(createMutation("3900", "EC2 ate my homework", "file://final_project.txt"));
    bw.flush();
    SamplerConfiguration sc1 = new SamplerConfiguration(RowSampler.class.getName());
    sc1.setOptions(ImmutableMap.of("hasher", "murmur3_32", "modulus", "3"));
    conn.tableOperations().setSamplerConfiguration(opts.getTableName(), sc1);
    Scanner scanner = conn.createScanner(opts.getTableName(), Authorizations.EMPTY);
    System.out.println("Scanning all data :");
    print(scanner);
    System.out.println();
    System.out.println("Scanning with sampler configuration.  Data was written before sampler was set on table, scan should fail.");
    scanner.setSamplerConfiguration(sc1);
    try {
        print(scanner);
    } catch (SampleNotPresentException e) {
        System.out.println("  Saw sample not present exception as expected.");
    }
    System.out.println();
    // compact table to recreate sample data
    conn.tableOperations().compact(opts.getTableName(), new CompactionConfig().setCompactionStrategy(NO_SAMPLE_STRATEGY));
    System.out.println("Scanning after compaction (compaction should have created sample data) : ");
    print(scanner);
    System.out.println();
    // update a document in the sample data
    bw.addMutation(createMutation("2317", "milk, eggs, bread, parmigiano-reggiano, butter", "file://groceries/9/txt"));
    bw.close();
    System.out.println("Scanning sample after updating content for docId 2317 (should see content change in sample data) : ");
    print(scanner);
    System.out.println();
    // change tables sampling configuration...
    SamplerConfiguration sc2 = new SamplerConfiguration(RowSampler.class.getName());
    sc2.setOptions(ImmutableMap.of("hasher", "murmur3_32", "modulus", "2"));
    conn.tableOperations().setSamplerConfiguration(opts.getTableName(), sc2);
    // compact table to recreate sample data using new configuration
    conn.tableOperations().compact(opts.getTableName(), new CompactionConfig().setCompactionStrategy(NO_SAMPLE_STRATEGY));
    System.out.println("Scanning with old sampler configuration.  Sample data was created using new configuration with a compaction.  Scan should fail.");
    try {
        // try scanning with old sampler configuration
        print(scanner);
    } catch (SampleNotPresentException e) {
        System.out.println("  Saw sample not present exception as expected ");
    }
    System.out.println();
    // update expected sampler configuration on scanner
    scanner.setSamplerConfiguration(sc2);
    System.out.println("Scanning with new sampler configuration : ");
    print(scanner);
    System.out.println();
}

Example 14 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testDropA.

@Test
public void testDropA() throws Exception {
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    writeFlush(c, tableName, "a");
    writeFlush(c, tableName, "b");
    // create a file that starts with A containing rows 'a' and 'b'
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    writeFlush(c, tableName, "c");
    writeFlush(c, tableName, "d");
    // drop files that start with A
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig(TestCompactionStrategy.class.getName());
    csConfig.setOptions(ImmutableMap.of("dropPrefix", "A", "inputPrefix", "F"));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(ImmutableSet.of("c", "d"), getRows(c, tableName));
    // this compaction should not drop files starting with A
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true));
    Assert.assertEquals(ImmutableSet.of("c", "d"), getRows(c, tableName));
}

Example 15 with CompactionConfig

use of org.apache.accumulo.core.client.admin.CompactionConfig in project accumulo by apache.

the class UserCompactionStrategyIT method testDropNone.

private void testDropNone(Map<String, String> options) throws Exception {
    Connector c = getConnector();
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    writeFlush(c, tableName, "a");
    writeFlush(c, tableName, "b");
    CompactionStrategyConfig csConfig = new CompactionStrategyConfig(TestCompactionStrategy.class.getName());
    csConfig.setOptions(options);
    c.tableOperations().compact(tableName, new CompactionConfig().setWait(true).setCompactionStrategy(csConfig));
    Assert.assertEquals(ImmutableSet.of("a", "b"), getRows(c, tableName));
}