Search in sources :

Example 21 with SortedMapIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator in project accumulo by apache.

the class MultiIteratorTest method test6.

@Test
public void test6() throws IOException {
    // TEst setting an endKey
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    newKeyValue(tm1, 3, 0, false, "1");
    newKeyValue(tm1, 4, 0, false, "2");
    newKeyValue(tm1, 6, 0, false, "3");
    List<SortedKeyValueIterator<Key, Value>> skvil = new ArrayList<>(1);
    skvil.add(new SortedMapIterator(tm1));
    MultiIterator mi = new MultiIterator(skvil, true);
    mi.seek(new Range(null, true, newKey(5, 9), false), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertEquals(mi.getTopKey(), newKey(3, 0));
    assertEquals("1", mi.getTopValue().toString());
    mi.next();
    assertTrue(mi.hasTop());
    assertEquals(mi.getTopKey(), newKey(4, 0));
    assertEquals("2", mi.getTopValue().toString());
    mi.next();
    assertFalse(mi.hasTop());
    mi.seek(new Range(newKey(4, 10), true, newKey(5, 9), false), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertEquals(mi.getTopKey(), newKey(4, 0));
    assertEquals("2", mi.getTopValue().toString());
    mi.next();
    assertFalse(mi.hasTop());
    mi.seek(new Range(newKey(4, 10), true, newKey(6, 0), false), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertEquals(mi.getTopKey(), newKey(4, 0));
    assertEquals("2", mi.getTopValue().toString());
    mi.next();
    assertFalse(mi.hasTop());
    mi.seek(new Range(newKey(4, 10), true, newKey(6, 0), true), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertEquals(mi.getTopKey(), newKey(4, 0));
    assertEquals("2", mi.getTopValue().toString());
    mi.next();
    assertTrue(mi.hasTop());
    assertEquals(mi.getTopKey(), newKey(6, 0));
    assertEquals("3", mi.getTopValue().toString());
    mi.next();
    assertFalse(mi.hasTop());
    mi.seek(new Range(newKey(4, 0), true, newKey(6, 0), false), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertEquals(mi.getTopKey(), newKey(4, 0));
    assertEquals("2", mi.getTopValue().toString());
    mi.next();
    assertFalse(mi.hasTop());
    mi.seek(new Range(newKey(4, 0), false, newKey(6, 0), false), EMPTY_COL_FAMS, false);
    assertFalse(mi.hasTop());
    mi.seek(new Range(newKey(4, 0), false, newKey(6, 0), true), EMPTY_COL_FAMS, false);
    assertTrue(mi.hasTop());
    assertEquals(mi.getTopKey(), newKey(6, 0));
    assertEquals("3", mi.getTopValue().toString());
    mi.next();
    assertFalse(mi.hasTop());
}
Also used : MultiIterator(org.apache.accumulo.core.iteratorsImpl.system.MultiIterator) Value(org.apache.accumulo.core.data.Value) ArrayList(java.util.ArrayList) SortedKeyValueIterator(org.apache.accumulo.core.iterators.SortedKeyValueIterator) TreeMap(java.util.TreeMap) SortedMapIterator(org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator) Range(org.apache.accumulo.core.data.Range) Key(org.apache.accumulo.core.data.Key) Test(org.junit.jupiter.api.Test)

Example 22 with SortedMapIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator in project accumulo by apache.

the class CombinerTest method testLossyOption.

/**
 * Tests the Lossy option will ignore errors in TypedValueCombiner. Uses SummingArrayCombiner to
 * generate error.
 */
@Test
public void testLossyOption() throws IOException {
    Encoder<List<Long>> encoder = new SummingArrayCombiner.VarLongArrayEncoder();
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    // keys that aggregate
    tm1.put(newKey(1, 1, 1, 1, false), new Value("badValue"));
    newKeyValue(tm1, 1, 1, 1, 2, false, nal(3L, 4L, 5L), encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, nal(), encoder);
    SummingArrayCombiner summingArrayCombiner = new SummingArrayCombiner();
    IteratorSetting iteratorSetting = new IteratorSetting(1, SummingArrayCombiner.class);
    SummingArrayCombiner.setEncodingType(iteratorSetting, SummingArrayCombiner.Type.VARLEN);
    Combiner.setColumns(iteratorSetting, Collections.singletonList(new IteratorSetting.Column("cf001")));
    // lossy = true so ignore bad value
    TypedValueCombiner.setLossyness(iteratorSetting, true);
    assertTrue(summingArrayCombiner.validateOptions(iteratorSetting.getOptions()));
    summingArrayCombiner.init(new SortedMapIterator(tm1), iteratorSetting.getOptions(), SCAN_IE);
    final Range range = new Range();
    summingArrayCombiner.seek(range, EMPTY_COL_FAMS, false);
    assertTrue(summingArrayCombiner.hasTop());
    assertEquals(newKey(1, 1, 1, 3), summingArrayCombiner.getTopKey());
    assertBytesEqual(encoder.encode(nal(3L, 4L, 5L)), summingArrayCombiner.getTopValue().get());
    summingArrayCombiner.next();
    assertFalse(summingArrayCombiner.hasTop());
    // lossy = false throw error for bad value
    TypedValueCombiner.setLossyness(iteratorSetting, false);
    assertTrue(summingArrayCombiner.validateOptions(iteratorSetting.getOptions()));
    summingArrayCombiner.init(new SortedMapIterator(tm1), iteratorSetting.getOptions(), SCAN_IE);
    assertThrows(ValueFormatException.class, () -> summingArrayCombiner.seek(range, EMPTY_COL_FAMS, false));
}
Also used : IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) Value(org.apache.accumulo.core.data.Value) ArrayList(java.util.ArrayList) List(java.util.List) TreeMap(java.util.TreeMap) SortedMapIterator(org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator) Range(org.apache.accumulo.core.data.Range) Key(org.apache.accumulo.core.data.Key) Test(org.junit.jupiter.api.Test)

Example 23 with SortedMapIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator in project accumulo by apache.

the class CombinerTest method test4.

@Test
public void test4() throws IOException {
    Encoder<Long> encoder = LongCombiner.STRING_ENCODER;
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    // keys that do not aggregate
    newKeyValue(tm1, 0, 0, 1, 1, false, 7L, encoder);
    // keys that aggregate
    newKeyValue(tm1, 1, 1, 1, 1, false, 2L, encoder);
    newKeyValue(tm1, 1, 1, 1, 2, false, 3L, encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, 4L, encoder);
    // keys that do not aggregate
    newKeyValue(tm1, 2, 2, 1, 1, false, 2L, encoder);
    newKeyValue(tm1, 2, 2, 1, 2, false, 3L, encoder);
    Combiner ai = new SummingCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(0, 0, 1, 1), ai.getTopKey());
    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 1), ai.getTopKey());
    assertEquals("2", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
    // seek test
    ai.seek(newRow(0, 0, 1, 0), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    // seek after key that aggregates
    ai.seek(newRow(1, 1, 1, 2), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
    // combine all columns
    is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
    Combiner.setCombineAllColumns(is, true);
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    ai.seek(new Range(), EMPTY_COL_FAMS, false);
    assertTrue(ai.hasTop());
    assertEquals(newKey(0, 0, 1, 1), ai.getTopKey());
    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(1, 1, 1, 3), ai.getTopKey());
    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertTrue(ai.hasTop());
    assertEquals(newKey(2, 2, 1, 2), ai.getTopKey());
    assertEquals("5", encoder.decode(ai.getTopValue().get()).toString());
    ai.next();
    assertFalse(ai.hasTop());
}
Also used : IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) Value(org.apache.accumulo.core.data.Value) LongCombiner(org.apache.accumulo.core.iterators.LongCombiner) TypedValueCombiner(org.apache.accumulo.core.iterators.TypedValueCombiner) Combiner(org.apache.accumulo.core.iterators.Combiner) TreeMap(java.util.TreeMap) SortedMapIterator(org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator) Range(org.apache.accumulo.core.data.Range) Key(org.apache.accumulo.core.data.Key) Test(org.junit.jupiter.api.Test)

Example 24 with SortedMapIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator in project accumulo by apache.

the class CombinerTest method test6.

@Test
public void test6() throws IOException {
    Encoder<Long> encoder = LongCombiner.VAR_LEN_ENCODER;
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    // keys that aggregate
    newKeyValue(tm1, 1, 1, 1, 1, false, 2L, encoder);
    newKeyValue(tm1, 1, 1, 1, 2, false, 3L, encoder);
    newKeyValue(tm1, 1, 1, 1, 3, false, 4L, encoder);
    Combiner ai = new SummingCombiner();
    IteratorSetting is = new IteratorSetting(1, SummingCombiner.class);
    LongCombiner.setEncodingType(is, VarLenEncoder.class.getName());
    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
    ai.init(new SortedMapIterator(tm1), is.getOptions(), SCAN_IE);
    // try seeking to the beginning of a key that aggregates
    ai.seek(newRow(1, 1, 1, 3, false), EMPTY_COL_FAMS, false);
    assertFalse(ai.hasTop());
}
Also used : IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) Value(org.apache.accumulo.core.data.Value) LongCombiner(org.apache.accumulo.core.iterators.LongCombiner) TypedValueCombiner(org.apache.accumulo.core.iterators.TypedValueCombiner) Combiner(org.apache.accumulo.core.iterators.Combiner) VarLenEncoder(org.apache.accumulo.core.iterators.LongCombiner.VarLenEncoder) TreeMap(java.util.TreeMap) SortedMapIterator(org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator) Key(org.apache.accumulo.core.data.Key) Test(org.junit.jupiter.api.Test)

Example 25 with SortedMapIterator

use of org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator in project accumulo by apache.

the class GrepIteratorTest method test.

@Test
public void test() throws IOException {
    GrepIterator gi = new GrepIterator();
    IteratorSetting is = new IteratorSetting(1, GrepIterator.class);
    GrepIterator.setTerm(is, "ab");
    gi.init(new SortedMapIterator(input), is.getOptions(), null);
    gi.seek(new Range(), EMPTY_COL_FAMS, false);
    checkEntries(gi, output);
    GrepIterator.setTerm(is, "cde");
    gi.init(new SortedMapIterator(input), is.getOptions(), null);
    gi.deepCopy(null);
    gi.seek(new Range(), EMPTY_COL_FAMS, false);
    checkEntries(gi, output);
    GrepIterator.setTerm(is, "def");
    gi.init(new SortedMapIterator(input), is.getOptions(), null);
    gi.seek(new Range(), EMPTY_COL_FAMS, false);
    checkEntries(gi, output);
}
Also used : IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) SortedMapIterator(org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator) Range(org.apache.accumulo.core.data.Range) Test(org.junit.jupiter.api.Test)

Aggregations

SortedMapIterator (org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator)109 Test (org.junit.jupiter.api.Test)97 Range (org.apache.accumulo.core.data.Range)93 Key (org.apache.accumulo.core.data.Key)89 Value (org.apache.accumulo.core.data.Value)89 TreeMap (java.util.TreeMap)82 IteratorSetting (org.apache.accumulo.core.client.IteratorSetting)38 Text (org.apache.hadoop.io.Text)29 PartialKey (org.apache.accumulo.core.data.PartialKey)16 Combiner (org.apache.accumulo.core.iterators.Combiner)12 LongCombiner (org.apache.accumulo.core.iterators.LongCombiner)12 TypedValueCombiner (org.apache.accumulo.core.iterators.TypedValueCombiner)12 ArrayList (java.util.ArrayList)11 HashSet (java.util.HashSet)9 DefaultIteratorEnvironment (org.apache.accumulo.core.iterators.DefaultIteratorEnvironment)8 ColumnFamilySkippingIterator (org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator)8 MultiIterator (org.apache.accumulo.core.iteratorsImpl.system.MultiIterator)8 ByteSequence (org.apache.accumulo.core.data.ByteSequence)7 SortedKeyValueIterator (org.apache.accumulo.core.iterators.SortedKeyValueIterator)7 SourceSwitchingIterator (org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator)7