Examples with NewTableConfiguration org.apache.accumulo.core.client.admin.NewTableConfiguration used on opensource projects

Example 6 with NewTableConfiguration

use of org.apache.accumulo.core.client.admin.NewTableConfiguration in project accumulo by apache.

the class TooManyDeletesIT method tooManyDeletesCompactionStrategyIT.

@Test
public void tooManyDeletesCompactionStrategyIT() throws Exception {
    Connector c = getConnector();
    String table = getUniqueNames(1)[0];
    SummarizerConfiguration sc = SummarizerConfiguration.builder(DeletesSummarizer.class).build();
    // TODO open issue about programatic config of compaction strategies
    NewTableConfiguration ntc = new NewTableConfiguration().enableSummarization(sc);
    HashMap<String, String> props = new HashMap<>();
    props.put(Property.TABLE_COMPACTION_STRATEGY.getKey(), TooManyDeletesCompactionStrategy.class.getName());
    props.put(Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey() + TooManyDeletesCompactionStrategy.THRESHOLD_OPT, ".25");
    // ensure compaction does not happen because of the number of files
    props.put(Property.TABLE_MAJC_RATIO.getKey(), "10");
    ntc.setProperties(props);
    c.tableOperations().create(table, ntc);
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        for (int i = 0; i < 1000; i++) {
            Mutation m = new Mutation("row" + i);
            m.put("f", "q", "v" + i);
            bw.addMutation(m);
        }
    }
    List<Summary> summaries = c.tableOperations().summaries(table).flush(true).withConfiguration(sc).retrieve();
    Assert.assertEquals(1, summaries.size());
    Summary summary = summaries.get(0);
    Assert.assertEquals(1000l, (long) summary.getStatistics().get(DeletesSummarizer.TOTAL_STAT));
    Assert.assertEquals(0l, (long) summary.getStatistics().get(DeletesSummarizer.DELETES_STAT));
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        for (int i = 0; i < 100; i++) {
            Mutation m = new Mutation("row" + i);
            m.putDelete("f", "q");
            bw.addMutation(m);
        }
    }
    summaries = c.tableOperations().summaries(table).flush(true).withConfiguration(sc).retrieve();
    Assert.assertEquals(1, summaries.size());
    summary = summaries.get(0);
    Assert.assertEquals(1100l, (long) summary.getStatistics().get(DeletesSummarizer.TOTAL_STAT));
    Assert.assertEquals(100l, (long) summary.getStatistics().get(DeletesSummarizer.DELETES_STAT));
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        for (int i = 100; i < 300; i++) {
            Mutation m = new Mutation("row" + i);
            m.putDelete("f", "q");
            bw.addMutation(m);
        }
    }
    // after a flush occurs Accumulo will check if a major compaction is needed. This check should call the compaction strategy, which should decide to compact
    // all files based on the number of deletes.
    c.tableOperations().flush(table, null, null, true);
    // wait for the compaction to happen
    while (true) {
        // the flush should cause
        summaries = c.tableOperations().summaries(table).flush(false).withConfiguration(sc).retrieve();
        Assert.assertEquals(1, summaries.size());
        summary = summaries.get(0);
        long total = summary.getStatistics().get(DeletesSummarizer.TOTAL_STAT);
        long deletes = summary.getStatistics().get(DeletesSummarizer.DELETES_STAT);
        if (total == 700 && deletes == 0) {
            // a compaction was triggered based on the number of deletes
            break;
        }
        UtilWaitThread.sleep(50);
    }
}

Example 7 with NewTableConfiguration

use of org.apache.accumulo.core.client.admin.NewTableConfiguration in project accumulo by apache.

the class TableOperationsImpl method create.

@Override
@Deprecated
public void create(String tableName, boolean limitVersion, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException {
    checkArgument(tableName != null, "tableName is null");
    checkArgument(timeType != null, "timeType is null");
    NewTableConfiguration ntc = new NewTableConfiguration().setTimeType(timeType);
    if (limitVersion)
        create(tableName, ntc);
    else
        create(tableName, ntc.withoutDefaultIterators());
}

Example 8 with NewTableConfiguration

use of org.apache.accumulo.core.client.admin.NewTableConfiguration in project accumulo by apache.

the class SampleIT method testBasic.

@Test
public void testBasic() throws Exception {
    Connector conn = getConnector();
    String tableName = getUniqueNames(1)[0];
    String clone = tableName + "_clone";
    conn.tableOperations().create(tableName, new NewTableConfiguration().enableSampling(SC1));
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    TreeMap<Key, Value> expected = new TreeMap<>();
    String someRow = writeData(bw, SC1, expected);
    Assert.assertEquals(20, expected.size());
    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
    Scanner isoScanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
    Scanner csiScanner = new ClientSideIteratorScanner(conn.createScanner(tableName, Authorizations.EMPTY));
    scanner.setSamplerConfiguration(SC1);
    csiScanner.setSamplerConfiguration(SC1);
    isoScanner.setSamplerConfiguration(SC1);
    isoScanner.setBatchSize(10);
    BatchScanner bScanner = conn.createBatchScanner(tableName, Authorizations.EMPTY, 2);
    bScanner.setSamplerConfiguration(SC1);
    bScanner.setRanges(Arrays.asList(new Range()));
    check(expected, scanner, bScanner, isoScanner, csiScanner);
    conn.tableOperations().flush(tableName, null, null, true);
    Scanner oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    check(expected, scanner, bScanner, isoScanner, csiScanner, oScanner);
    // ensure non sample data can be scanned after scanning sample data
    for (ScannerBase sb : Arrays.asList(scanner, bScanner, isoScanner, csiScanner, oScanner)) {
        sb.clearSamplerConfiguration();
        Assert.assertEquals(20000, Iterables.size(sb));
        sb.setSamplerConfiguration(SC1);
    }
    Iterator<Key> it = expected.keySet().iterator();
    while (it.hasNext()) {
        Key k = it.next();
        if (k.getRow().toString().equals(someRow)) {
            it.remove();
        }
    }
    expected.put(new Key(someRow, "cf1", "cq1", 8), new Value("42".getBytes()));
    expected.put(new Key(someRow, "cf1", "cq3", 8), new Value("suprise".getBytes()));
    Mutation m = new Mutation(someRow);
    m.put("cf1", "cq1", 8, "42");
    m.putDelete("cf1", "cq2", 8);
    m.put("cf1", "cq3", 8, "suprise");
    bw.addMutation(m);
    bw.close();
    check(expected, scanner, bScanner, isoScanner, csiScanner);
    conn.tableOperations().flush(tableName, null, null, true);
    oScanner = newOfflineScanner(conn, tableName, clone, SC1);
    check(expected, scanner, bScanner, isoScanner, csiScanner, oScanner);
    scanner.setRange(new Range(someRow));
    isoScanner.setRange(new Range(someRow));
    csiScanner.setRange(new Range(someRow));
    oScanner.setRange(new Range(someRow));
    bScanner.setRanges(Arrays.asList(new Range(someRow)));
    expected.clear();
    expected.put(new Key(someRow, "cf1", "cq1", 8), new Value("42".getBytes()));
    expected.put(new Key(someRow, "cf1", "cq3", 8), new Value("suprise".getBytes()));
    check(expected, scanner, bScanner, isoScanner, csiScanner, oScanner);
}

Example 9 with NewTableConfiguration

use of org.apache.accumulo.core.client.admin.NewTableConfiguration in project accumulo by apache.

the class SampleIT method testIterator.

@Test
public void testIterator() throws Exception {
    Connector conn = getConnector();
    String tableName = getUniqueNames(1)[0];
    String clone = tableName + "_clone";
    conn.tableOperations().create(tableName, new NewTableConfiguration().enableSampling(SC1));
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    TreeMap<Key, Value> expected = new TreeMap<>();
    writeData(bw, SC1, expected);
    ArrayList<Key> keys = new ArrayList<>(expected.keySet());
    Range range1 = new Range(keys.get(6), true, keys.get(11), true);
    Scanner scanner = null;
    Scanner isoScanner = null;
    ClientSideIteratorScanner csiScanner = null;
    BatchScanner bScanner = null;
    Scanner oScanner = null;
    try {
        scanner = conn.createScanner(tableName, Authorizations.EMPTY);
        isoScanner = new IsolatedScanner(conn.createScanner(tableName, Authorizations.EMPTY));
        csiScanner = new ClientSideIteratorScanner(conn.createScanner(tableName, Authorizations.EMPTY));
        bScanner = conn.createBatchScanner(tableName, Authorizations.EMPTY, 2);
        csiScanner.setIteratorSamplerConfiguration(SC1);
        List<? extends ScannerBase> scanners = Arrays.asList(scanner, isoScanner, bScanner, csiScanner);
        for (ScannerBase s : scanners) {
            s.addScanIterator(new IteratorSetting(100, IteratorThatUsesSample.class));
        }
        // the iterator should see less than 10 entries in sample data, and return data
        setRange(range1, scanners);
        for (ScannerBase s : scanners) {
            Assert.assertEquals(2954, countEntries(s));
        }
        Range range2 = new Range(keys.get(5), true, keys.get(18), true);
        setRange(range2, scanners);
        // the iterator should see more than 10 entries in sample data, and return no data
        for (ScannerBase s : scanners) {
            Assert.assertEquals(0, countEntries(s));
        }
        // flush an rerun same test against files
        conn.tableOperations().flush(tableName, null, null, true);
        oScanner = newOfflineScanner(conn, tableName, clone, null);
        oScanner.addScanIterator(new IteratorSetting(100, IteratorThatUsesSample.class));
        scanners = Arrays.asList(scanner, isoScanner, bScanner, csiScanner, oScanner);
        setRange(range1, scanners);
        for (ScannerBase s : scanners) {
            Assert.assertEquals(2954, countEntries(s));
        }
        setRange(range2, scanners);
        for (ScannerBase s : scanners) {
            Assert.assertEquals(0, countEntries(s));
        }
        updateSamplingConfig(conn, tableName, SC2);
        csiScanner.setIteratorSamplerConfiguration(SC2);
        oScanner = newOfflineScanner(conn, tableName, clone, null);
        oScanner.addScanIterator(new IteratorSetting(100, IteratorThatUsesSample.class));
        scanners = Arrays.asList(scanner, isoScanner, bScanner, csiScanner, oScanner);
        for (ScannerBase s : scanners) {
            try {
                countEntries(s);
                Assert.fail("Expected SampleNotPresentException, but it did not happen : " + s.getClass().getSimpleName());
            } catch (SampleNotPresentException e) {
            }
        }
    } finally {
        if (scanner != null) {
            scanner.close();
        }
        if (bScanner != null) {
            bScanner.close();
        }
        if (isoScanner != null) {
            isoScanner.close();
        }
        if (csiScanner != null) {
            csiScanner.close();
        }
        if (oScanner != null) {
            oScanner.close();
        }
    }
}

Example 10 with NewTableConfiguration

use of org.apache.accumulo.core.client.admin.NewTableConfiguration in project accumulo by apache.

the class ConditionalWriterIT method testIterators.

@Test
public void testIterators() throws Exception {
    Connector conn = getConnector();
    String tableName = getUniqueNames(1)[0];
    conn.tableOperations().create(tableName, new NewTableConfiguration().withoutDefaultIterators());
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    Mutation m = new Mutation("ACCUMULO-1000");
    m.put("count", "comments", "1");
    bw.addMutation(m);
    bw.addMutation(m);
    bw.addMutation(m);
    m = new Mutation("ACCUMULO-1001");
    m.put("count2", "comments", "1");
    bw.addMutation(m);
    bw.addMutation(m);
    m = new Mutation("ACCUMULO-1002");
    m.put("count2", "comments", "1");
    bw.addMutation(m);
    bw.addMutation(m);
    bw.close();
    IteratorSetting iterConfig = new IteratorSetting(10, SummingCombiner.class);
    SummingCombiner.setEncodingType(iterConfig, Type.STRING);
    SummingCombiner.setColumns(iterConfig, Collections.singletonList(new IteratorSetting.Column("count")));
    IteratorSetting iterConfig2 = new IteratorSetting(10, SummingCombiner.class);
    SummingCombiner.setEncodingType(iterConfig2, Type.STRING);
    SummingCombiner.setColumns(iterConfig2, Collections.singletonList(new IteratorSetting.Column("count2", "comments")));
    IteratorSetting iterConfig3 = new IteratorSetting(5, VersioningIterator.class);
    VersioningIterator.setMaxVersions(iterConfig3, 1);
    try (Scanner scanner = conn.createScanner(tableName, new Authorizations())) {
        scanner.addScanIterator(iterConfig);
        scanner.setRange(new Range("ACCUMULO-1000"));
        scanner.fetchColumn(new Text("count"), new Text("comments"));
        Entry<Key, Value> entry = Iterables.getOnlyElement(scanner);
        Assert.assertEquals("3", entry.getValue().toString());
        try (ConditionalWriter cw = conn.createConditionalWriter(tableName, new ConditionalWriterConfig())) {
            ConditionalMutation cm0 = new ConditionalMutation("ACCUMULO-1000", new Condition("count", "comments").setValue("3"));
            cm0.put("count", "comments", "1");
            Assert.assertEquals(Status.REJECTED, cw.write(cm0).getStatus());
            entry = Iterables.getOnlyElement(scanner);
            Assert.assertEquals("3", entry.getValue().toString());
            ConditionalMutation cm1 = new ConditionalMutation("ACCUMULO-1000", new Condition("count", "comments").setIterators(iterConfig).setValue("3"));
            cm1.put("count", "comments", "1");
            Assert.assertEquals(Status.ACCEPTED, cw.write(cm1).getStatus());
            entry = Iterables.getOnlyElement(scanner);
            Assert.assertEquals("4", entry.getValue().toString());
            ConditionalMutation cm2 = new ConditionalMutation("ACCUMULO-1000", new Condition("count", "comments").setValue("4"));
            cm2.put("count", "comments", "1");
            Assert.assertEquals(Status.REJECTED, cw.write(cm1).getStatus());
            entry = Iterables.getOnlyElement(scanner);
            Assert.assertEquals("4", entry.getValue().toString());
            // run test with multiple iterators passed in same batch and condition with two iterators
            ConditionalMutation cm3 = new ConditionalMutation("ACCUMULO-1000", new Condition("count", "comments").setIterators(iterConfig).setValue("4"));
            cm3.put("count", "comments", "1");
            ConditionalMutation cm4 = new ConditionalMutation("ACCUMULO-1001", new Condition("count2", "comments").setIterators(iterConfig2).setValue("2"));
            cm4.put("count2", "comments", "1");
            ConditionalMutation cm5 = new ConditionalMutation("ACCUMULO-1002", new Condition("count2", "comments").setIterators(iterConfig2, iterConfig3).setValue("2"));
            cm5.put("count2", "comments", "1");
            Iterator<Result> results = cw.write(Arrays.asList(cm3, cm4, cm5).iterator());
            Map<String, Status> actual = new HashMap<>();
            while (results.hasNext()) {
                Result result = results.next();
                String k = new String(result.getMutation().getRow());
                Assert.assertFalse("Did not expect to see multiple resultus for the row: " + k, actual.containsKey(k));
                actual.put(k, result.getStatus());
            }
            Map<String, Status> expected = new HashMap<>();
            expected.put("ACCUMULO-1000", Status.ACCEPTED);
            expected.put("ACCUMULO-1001", Status.ACCEPTED);
            expected.put("ACCUMULO-1002", Status.REJECTED);
            Assert.assertEquals(expected, actual);
        }
    }
}