Examples with Combiner org.apache.accumulo.core.iterators.Combiner used on opensource projects

Example 6 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method sumArray.

public static void sumArray(Class<? extends Encoder<List<Long>>> encoderClass, SummingArrayCombiner.Type type) throws IOException, InstantiationException, IllegalAccessException {
    Encoder<List<Long>> encoder = encoderClass.newInstance();
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    // keys that aggregate
    newKeyValue(tm1, 1, 1, 1, 1, false, nal(1l, 2l), encoder);
    newKeyValue(tm1, 1, 1, 1, 2, false, nal(3l, 4l, 5l), encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, nal(), encoder);
    Combiner ai = new SummingArrayCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingArrayCombiner.class);
    SummingArrayCombiner.setEncodingType(is, type);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertBytesEqual(encoder.encode(nal(4l, 6l, 5l)), ai.getTopValue().get());
    ai.next();
    assertFalse(ai.hasTop());
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, encoderClass);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertBytesEqual(encoder.encode(nal(4l, 6l, 5l)), ai.getTopValue().get());
    ai.next();
    assertFalse(ai.hasTop());
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, encoderClass.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertBytesEqual(encoder.encode(nal(4l, 6l, 5l)), ai.getTopValue().get());
    ai.next();
    assertFalse(ai.hasTop());
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, SummingCombiner.VAR_LEN_ENCODER.getClass().getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    try {
        ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }
    is.clearOptions();
    SummingArrayCombiner.setEncodingType(is, BadEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    try {
        ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
        Assert.fail();
    } catch (IllegalArgumentException e) {
    }
}

Example 7 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method test5.

@Test
public void test5() throws IOException {
    Encoder<Long> encoder = LongCombiner.STRING_ENCODER;
    // try aggregating across multiple data sets that contain
    // the exact same keys w/ different values
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    newKeyValue(tm1, 1, 1, 1, 1, false, 2l, encoder);
    TreeMap<Key, Value> tm2 = new TreeMap<>();
    newKeyValue(tm2, 1, 1, 1, 1, false, 3l, encoder);
    TreeMap<Key, Value> tm3 = new TreeMap<>();
    newKeyValue(tm3, 1, 1, 1, 1, false, 4l, encoder);
    Combiner ai = new SummingCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, StringEncoder.class);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    List<SortedKeyValueIterator<Key, Value>> sources = new ArrayList<>(3);
    sources.add(new SortedMapIterator(tm1));
    sources.add(new SortedMapIterator(tm2));
    sources.add(new SortedMapIterator(tm3));
    MultiIterator mi = new MultiIterator(sources, true);
    ai.init(mi, is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 1), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
}

Example 8 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method sumAllColumns.

@Test
public void sumAllColumns() throws IOException {
    TreeMap<Key, Value> tm = new TreeMap<>();
    tm.put(new Key("r", "count", "a", 1), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "a", 2), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "b", 3), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "b", 4), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "b", 5), new Value("1".getBytes()));
    tm.put(new Key("r", "count", "c", 6), new Value("1".getBytes()));
    SortedMapIterator smi = new SortedMapIterator(tm);
    Combiner iter = new SummingCombiner();
    IteratorSetting s = new IteratorSetting(10, "s", SummingCombiner.class);
    SummingCombiner.setColumns(s, Collections.singletonList(new IteratorSetting.Column("count")));
    SummingCombiner.setEncodingType(s, LongCombiner.StringEncoder.class);
    iter.init(smi, s.getOptions(), SCAN_IE);
    Combiner iter2 = new SummingCombiner();
    IteratorSetting s2 = new IteratorSetting(10, "s2", SummingCombiner.class);
    SummingCombiner.setColumns(s2, Collections.singletonList(new IteratorSetting.Column("count", "a")));
    SummingCombiner.setEncodingType(s2, LongCombiner.StringEncoder.class);
    iter2.init(iter, s.getOptions(), SCAN_IE);
    iter2.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(iter2.hasTop());
    assertEquals("2", iter2.getTopValue().toString());
    iter2.next();
    assertTrue(iter2.hasTop());
    assertEquals("3", iter2.getTopValue().toString());
    iter2.next();
    assertTrue(iter2.hasTop());
    assertEquals("1", iter2.getTopValue().toString());
    iter2.next();
    assertFalse(iter2.hasTop());
}

Example 9 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method test2.

@Test
public void test2() throws IOException {
    Encoder<Long> encoder = LongCombiner.VAR_LEN_ENCODER;
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    // keys that aggregate
    newKeyValue(tm1, 1, 1, 1, 1, false, 2l, encoder);
    newKeyValue(tm1, 1, 1, 1, 2, false, 3l, encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, 4l, encoder);
    Combiner ai = new SummingCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, VarLenEncoder.class);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
    // try seeking to the beginning of a key that aggregates
    ai.seek(newRow(1, 1, 1, 3), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
    // try seeking the middle of a key the aggregates
    ai.seek(newRow(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    assertFalse(ai.hasTop());
    // try seeking to the end of a key the aggregates
    ai.seek(newRow(1, 1, 1, 1), EMPTY_COL_FAMS, false);
    assertFalse(ai.hasTop());
    // try seeking before a key the aggregates
    ai.seek(newRow(1, 1, 1, 4), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
}

Example 10 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method test3.

@Test
public void test3() throws IOException {
    Encoder<Long> encoder = LongCombiner.FIXED_LEN_ENCODER;
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    // keys that aggregate
    newKeyValue(tm1, 1, 1, 1, 1, false, 2l, encoder);
    newKeyValue(tm1, 1, 1, 1, 2, false, 3l, encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, 4l, encoder);
    // keys that do not aggregate
    newKeyValue(tm1, 2, 2, 1, 1, false, 2l, encoder);
    newKeyValue(tm1, 2, 2, 1, 2, false, 3l, encoder);
    Combiner ai = new SummingCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, FixedLenEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
    // seek after key that aggregates
    ai.seek(newRow(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    // seek before key that aggregates
    ai.seek(newRow(1, 1, 1, 4), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
}