Examples with Combiner org.apache.accumulo.core.iterators.Combiner used on opensource projects

Example 1 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method test7.

@Test
public void test7() throws IOException {
    Encoder<Long> encoder = LongCombiner.FIXED_LEN_ENCODER;
    // test that delete is not aggregated
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    newKeyValue(tm1, 1, 1, 1, 2, true, 0l, encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, 4l, encoder);
    newKeyValue(tm1, 1, 1, 1, 4, false, 3l, encoder);
    Combiner ai = new SummingCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.FIXEDLEN);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(newRow(1, 1, 1, 4, true), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 4), ai.getTopKey());
    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 2, true), ai.getTopKey());
    assertEquals("0", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
    tm1 = new TreeMap<>();
    newKeyValue(tm1, 1, 1, 1, 2, true, 0l, encoder);
    ai = new SummingCombiner();
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(newRow(1, 1, 1, 4, true), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 2, true), ai.getTopKey());
    assertEquals("0", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
}

Example 2 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method test6.

@Test
public void test6() throws IOException {
    Encoder<Long> encoder = LongCombiner.VAR_LEN_ENCODER;
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    // keys that aggregate
    newKeyValue(tm1, 1, 1, 1, 1, false, 2l, encoder);
    newKeyValue(tm1, 1, 1, 1, 2, false, 3l, encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, 4l, encoder);
    Combiner ai = new SummingCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, VarLenEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    // try seeking to the beginning of a key that aggregates
    ai.seek(newRow(1, 1, 1, 3, false), EMPTY_COL_FAMS, false);
    assertFalse(ai.hasTop());
}

Example 3 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method test4.

@Test
public void test4() throws IOException {
    Encoder<Long> encoder = LongCombiner.STRING_ENCODER;
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    // keys that do not aggregate
    newKeyValue(tm1, 0, 0, 1, 1, false, 7l, encoder);
    // keys that aggregate
    newKeyValue(tm1, 1, 1, 1, 1, false, 2l, encoder);
    newKeyValue(tm1, 1, 1, 1, 2, false, 3l, encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, 4l, encoder);
    // keys that do not aggregate
    newKeyValue(tm1, 2, 2, 1, 1, false, 2l, encoder);
    newKeyValue(tm1, 2, 2, 1, 2, false, 3l, encoder);
    Combiner ai = new SummingCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(0, 0, 1, 1), ai.getTopKey());
    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
    // seek test
    ai.seek(newRow(0, 0, 1, 0), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    // seek after key that aggregates
    ai.seek(newRow(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    // combine all columns
    is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
    Combiner.setCombineAllColumns(is, true);
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(0, 0, 1, 1), ai.getTopKey());
    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("5", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
}

Example 4 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method testDeepCopy.

@Test
public void testDeepCopy() throws IOException {
    Encoder<Long> encoder = LongCombiner.FIXED_LEN_ENCODER;
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    // keys that aggregate
    newKeyValue(tm1, 1, 1, 1, 1, false, 2l, encoder);
    newKeyValue(tm1, 1, 1, 1, 2, false, 3l, encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, 4l, encoder);
    // keys that do not aggregate
    newKeyValue(tm1, 2, 2, 1, 1, false, 2l, encoder);
    newKeyValue(tm1, 2, 2, 1, 2, false, 3l, encoder);
    Combiner ai = new SummingCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, FixedLenEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    SortedKeyValueIterator<Key, Value> ai2 = ai.deepCopy(null);
    SortedKeyValueIterator<Key, Value> ai3 = ai.deepCopy(null);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
    // seek after key that aggregates
    ai2.seek(newRow(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    assertTrue(ai2.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai2.getTopKey());
    assertEquals("3", encoder.decode(ai2.getTopValue().get()).toString());
    // seek before key that aggregates
    ai3.seek(newRow(1, 1, 1, 4), EMPTY_COL_FAMS, false);
    assertTrue(ai3.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai3.getTopKey());
    assertEquals("9", encoder.decode(ai3.getTopValue().get()).toString());
    ai3.next();
    assertTrue(ai3.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai3.getTopKey());
    assertEquals("3", encoder.decode(ai3.getTopValue().get()).toString());
}

Example 5 with Combiner

use of org.apache.accumulo.core.iterators.Combiner in project accumulo by apache.

the class CombinerTest method runDeleteHandlingTest.

private void runDeleteHandlingTest(TreeMap<Key, Value> input, TreeMap<Key, Value> expected, Boolean rofco, IteratorEnvironment env, String expectedLog, boolean clearLogMsgCache) throws Exception {
    boolean deepCopy = expected == null;
    if (clearLogMsgCache) {
        CombinerTestUtil.clearLogCache();
    }
    StringWriter writer = new StringWriter();
    WriterAppender appender = new WriterAppender(new PatternLayout("%p, %m%n"), writer);
    Logger logger = Logger.getLogger(Combiner.class);
    boolean additivity = logger.getAdditivity();
    try {
        logger.addAppender(appender);
        logger.setAdditivity(false);
        Combiner ai = new SummingCombiner();
        IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
        SummingCombiner.setEncodingType(is, LongCombiner.StringEncoder.class);
        Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
        if (rofco != null) {
            Combiner.setReduceOnFullCompactionOnly(is, rofco);
        }
        ai.init(new SortedMapIterator(input), is.getOptions(), env);
        if (deepCopy)
            assertEquals(expected, readAll(ai.deepCopy(env)));
        assertEquals(expected, readAll(ai));
    } finally {
        logger.removeAppender(appender);
        logger.setAdditivity(additivity);
    }
    String logMsgs = writer.toString();
    if (expectedLog == null) {
        Assert.assertTrue("Expected 0 length log message, but got : " + logMsgs, logMsgs.length() == 0);
    } else {
        logMsgs = logMsgs.replace('\n', ' ');
        Assert.assertTrue("Did not match pattern [" + expectedLog + "] in [" + logMsgs + "]", logMsgs.matches(expectedLog));
    }
}