Examples with ColumnFamilySkippingIterator org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator used on opensource projects

Example 6 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class ColumnFamilySkippingIteratorTest method test2.

@Test
public void test2() throws Exception {
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    for (int r = 0; r < 10; r++) {
        for (int cf = 0; cf < 1000; cf++) {
            for (int cq = 0; cq < 3; cq++) {
                put(tm1, r, cf, cq, 6, r * cf * cq);
            }
        }
    }
    HashSet<ByteSequence> allColfams = new HashSet<>();
    for (int cf = 0; cf < 1000; cf++) {
        allColfams.add(new ArrayByteSequence(String.format("%06d", cf)));
    }
    ColumnFamilySkippingIterator cfi = new ColumnFamilySkippingIterator(new SortedMapIterator(tm1));
    HashSet<ByteSequence> colfams = new HashSet<>();
    runTest(cfi, 30000, 0, allColfams, colfams);
    colfams.add(new ArrayByteSequence(String.format("%06d", 60)));
    runTest(cfi, 30000, 30, allColfams, colfams);
    colfams.add(new ArrayByteSequence(String.format("%06d", 602)));
    runTest(cfi, 30000, 60, allColfams, colfams);
    colfams.add(new ArrayByteSequence(String.format("%06d", 0)));
    runTest(cfi, 30000, 90, allColfams, colfams);
    colfams.add(new ArrayByteSequence(String.format("%06d", 999)));
    runTest(cfi, 30000, 120, allColfams, colfams);
    colfams.remove(new ArrayByteSequence(String.format("%06d", 0)));
    runTest(cfi, 30000, 90, allColfams, colfams);
    colfams.add(new ArrayByteSequence(String.format("%06d", 1000)));
    runTest(cfi, 30000, 90, allColfams, colfams);
    colfams.remove(new ArrayByteSequence(String.format("%06d", 999)));
    runTest(cfi, 30000, 60, allColfams, colfams);
    colfams.add(new ArrayByteSequence(String.format("%06d", 61)));
    runTest(cfi, 30000, 90, allColfams, colfams);
    for (int i = 62; i < 100; i++) colfams.add(new ArrayByteSequence(String.format("%06d", i)));
    runTest(cfi, 30000, 1230, allColfams, colfams);
}

Example 7 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class RFileTest method runVersionTest.

private void runVersionTest(int version, ConfigurationCopy aconf) throws Exception {
    InputStream in = this.getClass().getClassLoader().getResourceAsStream("org/apache/accumulo/core/file/rfile/ver_" + version + ".rf");
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    byte[] buf = new byte[1024];
    int read;
    while ((read = in.read(buf)) > 0) baos.write(buf, 0, read);
    byte[] data = baos.toByteArray();
    SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data);
    FSDataInputStream in2 = new FSDataInputStream(bais);
    aconf.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    aconf.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(100000));
    aconf.set(Property.TSERV_DATACACHE_SIZE, Long.toString(100000000));
    aconf.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(100000000));
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(aconf);
    manager.start(new BlockCacheConfiguration(aconf));
    CachableBuilder cb = new CachableBuilder().input(in2, "cache-1").length(data.length).conf(hadoopConf).cryptoService(CryptoServiceFactory.newInstance(aconf, ClassloaderType.JAVA)).cacheProvider(new BasicCacheProvider(manager.getBlockCache(CacheType.INDEX), manager.getBlockCache(CacheType.DATA)));
    Reader reader = new RFile.Reader(cb);
    checkIndex(reader);
    ColumnFamilySkippingIterator iter = new ColumnFamilySkippingIterator(reader);
    for (int start : new int[] { 0, 10, 100, 998 }) {
        for (int cf = 1; cf <= 4; cf++) {
            if (start == 0)
                iter.seek(new Range(), newColFamByteSequence(formatString("cf_", cf)), true);
            else
                iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(formatString("cf_", cf)), true);
            for (int i = start; i < 1000; i++) {
                assertTrue(iter.hasTop());
                assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey());
                assertEquals(newValue(i + ""), iter.getTopValue());
                iter.next();
            }
            assertFalse(iter.hasTop());
        }
        if (start == 0)
            iter.seek(new Range(), newColFamByteSequence(), false);
        else
            iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(), false);
        for (int i = start; i < 1000; i++) {
            for (int cf = 1; cf <= 4; cf++) {
                assertTrue(iter.hasTop());
                assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey());
                assertEquals(newValue(i + ""), iter.getTopValue());
                iter.next();
            }
        }
        assertFalse(iter.hasTop());
    }
    manager.stop();
    reader.close();
}

Example 8 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class RowFilterTest method test1.

@Test
public void test1() throws Exception {
    ColumnFamilySkippingIterator source = new ColumnFamilySkippingIterator(new SortedMapIterator(createKeyValues()));
    RowFilter filter = new SummingRowFilter();
    filter.init(source, Collections.emptyMap(), new DefaultIteratorEnvironment());
    filter.seek(new Range(), Collections.emptySet(), false);
    assertEquals(Set.of("2", "3"), getRows(filter));
    ByteSequence cf = new ArrayByteSequence("cf2");
    filter.seek(new Range(), Set.of(cf), true);
    assertEquals(Set.of("1", "3", "0", "4"), getRows(filter));
    filter.seek(new Range("0", "4"), Collections.emptySet(), false);
    assertEquals(Set.of("2", "3"), getRows(filter));
    filter.seek(new Range("2"), Collections.emptySet(), false);
    assertEquals(Set.of("2"), getRows(filter));
    filter.seek(new Range("4"), Collections.emptySet(), false);
    assertEquals(Set.of(), getRows(filter));
    filter.seek(new Range("4"), Set.of(cf), true);
    assertEquals(Set.of("4"), getRows(filter));
}

Example 9 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class TransformingIteratorTest method setUpTransformIterator.

private void setUpTransformIterator(Class<? extends TransformingIterator> clazz, boolean setupAuths) throws IOException {
    SortedMapIterator source = new SortedMapIterator(data);
    ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(source);
    SortedKeyValueIterator<Key, Value> visFilter = VisibilityFilter.wrap(cfsi, authorizations, new byte[0]);
    ReuseIterator reuserIter = new ReuseIterator();
    reuserIter.init(visFilter, EMPTY_OPTS, null);
    try {
        titer = clazz.getDeclaredConstructor().newInstance();
    } catch (ReflectiveOperationException e) {
        throw new RuntimeException(e);
    }
    IteratorEnvironment iterEnv = EasyMock.createMock(IteratorEnvironment.class);
    EasyMock.expect(iterEnv.getIteratorScope()).andReturn(IteratorScope.scan).anyTimes();
    EasyMock.replay(iterEnv);
    Map<String, String> opts;
    if (setupAuths) {
        IteratorSetting cfg = new IteratorSetting(21, clazz);
        TransformingIterator.setAuthorizations(cfg, new Authorizations("vis0", "vis1", "vis2", "vis3"));
        opts = cfg.getOptions();
    } else {
        opts = Map.of();
    }
    titer.init(reuserIter, opts, iterEnv);
}

Example 10 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class RowDeletingIteratorTest method test3.

@Test
public void test3() throws Exception {
    TreeMap<Key, Value> tm1 = new TreeMap<>();
    put(tm1, "r1", "", "", 10, RowDeletingIterator.DELETE_ROW_VALUE);
    put(tm1, "r1", "", "cq1", 5, "v1");
    put(tm1, "r1", "cf1", "cq1", 5, "v1");
    put(tm1, "r2", "", "cq1", 5, "v1");
    put(tm1, "r2", "cf1", "cq1", 5, "v1");
    RowDeletingIterator rdi = new RowDeletingIterator();
    rdi.init(new ColumnFamilySkippingIterator(new SortedMapIterator(tm1)), null, new TestIE(IteratorScope.scan, false));
    HashSet<ByteSequence> cols = new HashSet<>();
    cols.add(new ArrayByteSequence("cf1".getBytes()));
    rdi.seek(new Range(), cols, true);
    testAssertions(rdi, "r2", "cf1", "cq1", 5, "v1");
    cols.clear();
    cols.add(new ArrayByteSequence("".getBytes()));
    rdi.seek(new Range(), cols, false);
    testAssertions(rdi, "r2", "cf1", "cq1", 5, "v1");
    cols.clear();
    rdi.seek(new Range(), cols, false);
    testAssertions(rdi, "r2", "", "cq1", 5, "v1");
    rdi.next();
    testAssertions(rdi, "r2", "cf1", "cq1", 5, "v1");
}