Examples with Filter org.apache.accumulo.core.iterators.Filter used on opensource projects

Example 1 with Filter

use of org.apache.accumulo.core.iterators.Filter in project accumulo by apache.

the class FilterTest method test1.

@Test
public void test1() throws IOException {
    Text colf = new Text("a");
    Text colq = new Text("b");
    Value dv = new Value();
    TreeMap<Key, Value> tm = new TreeMap<>();
    for (int i = 0; i < 1000; i++) {
        Key k = new Key(new Text(String.format("%03d", i)), colf, colq);
        tm.put(k, dv);
    }
    assertEquals(1000, tm.size());
    Filter filter1 = new SimpleFilter();
    filter1.init(new SortedMapIterator(tm), EMPTY_OPTS, null);
    filter1.seek(new Range(), EMPTY_COL_FAMS, false);
    int size = size(filter1);
    assertEquals(100, size);
    Filter fi = new SimpleFilter();
    fi.init(new SortedMapIterator(tm), EMPTY_OPTS, null);
    Key k = new Key(new Text("500"));
    fi.seek(new Range(k, null), EMPTY_COL_FAMS, false);
    size = size(fi);
    assertEquals(50, size);
    filter1 = new SimpleFilter();
    filter1.init(new SortedMapIterator(tm), EMPTY_OPTS, null);
    Filter filter2 = new SimpleFilter2();
    filter2.init(filter1, EMPTY_OPTS, null);
    filter2.seek(new Range(), EMPTY_COL_FAMS, false);
    size = size(filter2);
    assertEquals(0, size);
}

Example 2 with Filter

use of org.apache.accumulo.core.iterators.Filter in project accumulo by apache.

the class FilterTest method testNoVisFilter.

@Test
public void testNoVisFilter() throws IOException {
    TreeMap<Key, Value> tm = new TreeMap<>();
    Value v = new Value();
    for (int i = 0; i < 1000; i++) {
        Key k = new Key(String.format("%03d", i), "a", "b", i % 10 == 0 ? "vis" : "");
        tm.put(k, v);
    }
    assertEquals(1000, tm.size());
    Filter filter = new ReqVisFilter();
    filter.init(new SortedMapIterator(tm), EMPTY_OPTS, null);
    filter.seek(new Range(), EMPTY_COL_FAMS, false);
    int size = size(filter);
    assertEquals(100, size);
}

Example 3 with Filter

use of org.apache.accumulo.core.iterators.Filter in project accumulo by apache.

the class FilterTest method testDeletes.

@Test
public void testDeletes() throws IOException {
    Text colf = new Text("a");
    Text colq = new Text("b");
    Value dv = new Value();
    TreeMap<Key, Value> tm = new TreeMap<>();
    Key k = new Key(new Text("0"), colf, colq);
    tm.put(k, dv);
    k = new Key(new Text("1"), colf, colq, 10);
    k.setDeleted(true);
    tm.put(k, dv);
    k = new Key(new Text("1"), colf, colq, 5);
    tm.put(k, dv);
    k = new Key(new Text("10"), colf, colq);
    tm.put(k, dv);
    assertEquals(4, tm.size());
    Filter filter = new SimpleFilter();
    filter.init(new SortedMapIterator(tm), EMPTY_OPTS, null);
    filter.seek(new Range(), EMPTY_COL_FAMS, false);
    int size = size(filter);
    assertEquals(3, size);
}

Example 4 with Filter

use of org.apache.accumulo.core.iterators.Filter in project accumulo by apache.

the class FilterTest method test1neg.

@Test
public void test1neg() throws IOException {
    Text colf = new Text("a");
    Text colq = new Text("b");
    Value dv = new Value();
    TreeMap<Key, Value> tm = new TreeMap<>();
    for (int i = 0; i < 1000; i++) {
        Key k = new Key(new Text(String.format("%03d", i)), colf, colq);
        tm.put(k, dv);
    }
    assertEquals(1000, tm.size());
    Filter filter = new SimpleFilter();
    IteratorSetting is = new IteratorSetting(1, SimpleFilter.class);
    Filter.setNegate(is, true);
    filter.init(new SortedMapIterator(tm), is.getOptions(), null);
    filter.seek(new Range(), EMPTY_COL_FAMS, false);
    int size = size(filter);
    assertEquals(900, size);
    filter.init(new SortedMapIterator(tm), is.getOptions(), null);
    Key k = new Key(new Text("500"));
    filter.seek(new Range(k, null), EMPTY_COL_FAMS, false);
    size = size(filter);
    assertEquals(450, size);
    filter.init(new SortedMapIterator(tm), EMPTY_OPTS, null);
    Filter filter2 = new SimpleFilter2();
    filter2.init(filter, is.getOptions(), null);
    filter2.seek(new Range(), EMPTY_COL_FAMS, false);
    size = size(filter2);
    assertEquals(100, size);
}

Example 5 with Filter

use of org.apache.accumulo.core.iterators.Filter in project accumulo by apache.

the class SummaryIT method compactionTest.

@Test
public void compactionTest() throws Exception {
    final String table = getUniqueNames(1)[0];
    Connector c = getConnector();
    NewTableConfiguration ntc = new NewTableConfiguration();
    SummarizerConfiguration sc1 = SummarizerConfiguration.builder(FooCounter.class.getName()).build();
    ntc.enableSummarization(sc1);
    c.tableOperations().create(table, ntc);
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "bar1", "f1", "q1", "v1");
        write(bw, "bar2", "f1", "q1", "v2");
        write(bw, "foo1", "f1", "q1", "v3");
    }
    // Create a compaction config that will filter out foos if there are too many. Uses summary data to know if there are too many foos.
    CompactionStrategyConfig csc = new CompactionStrategyConfig(FooCS.class.getName());
    List<IteratorSetting> iterators = Collections.singletonList(new IteratorSetting(100, FooFilter.class));
    CompactionConfig compactConfig = new CompactionConfig().setFlush(true).setCompactionStrategy(csc).setIterators(iterators).setWait(true);
    // this compaction should make no changes because there are less foos than bars
    c.tableOperations().compact(table, compactConfig);
    try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
        Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
        Map<String, Long> counts = // convert to row
        stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
        r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
        groupingBy(identity(), counting()));
        Assert.assertEquals(1l, (long) counts.getOrDefault("foo", 0l));
        Assert.assertEquals(2l, (long) counts.getOrDefault("bar", 0l));
        Assert.assertEquals(2, counts.size());
    }
    try (BatchWriter bw = c.createBatchWriter(table, new BatchWriterConfig())) {
        write(bw, "foo2", "f1", "q1", "v4");
        write(bw, "foo3", "f1", "q1", "v5");
        write(bw, "foo4", "f1", "q1", "v6");
    }
    // this compaction should remove all foos because there are more foos than bars
    c.tableOperations().compact(table, compactConfig);
    try (Scanner scanner = c.createScanner(table, Authorizations.EMPTY)) {
        Stream<Entry<Key, Value>> stream = StreamSupport.stream(scanner.spliterator(), false);
        Map<String, Long> counts = // convert to row
        stream.map(e -> e.getKey().getRowData().toString()).map(// strip numbers off row
        r -> r.replaceAll("[0-9]+", "")).collect(// count different row types
        groupingBy(identity(), counting()));
        Assert.assertEquals(0l, (long) counts.getOrDefault("foo", 0l));
        Assert.assertEquals(2l, (long) counts.getOrDefault("bar", 0l));
        Assert.assertEquals(1, counts.size());
    }
}