Examples with ColumnFamilySkippingIterator org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator used on opensource projects

Example 1 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class RFileTest method runVersionTest.

private void runVersionTest(int version) throws IOException {
    InputStream in = this.getClass().getClassLoader().getResourceAsStream("org/apache/accumulo/core/file/rfile/ver_" + version + ".rf");
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    byte[] buf = new byte[1024];
    int read;
    while ((read = in.read(buf)) > 0) baos.write(buf, 0, read);
    byte[] data = baos.toByteArray();
    SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data);
    FSDataInputStream in2 = new FSDataInputStream(bais);
    AccumuloConfiguration aconf = DefaultConfiguration.getInstance();
    CachableBlockFile.Reader _cbr = new CachableBlockFile.Reader(in2, data.length, CachedConfiguration.getInstance(), aconf);
    Reader reader = new RFile.Reader(_cbr);
    checkIndex(reader);
    ColumnFamilySkippingIterator iter = new ColumnFamilySkippingIterator(reader);
    for (int start : new int[] { 0, 10, 100, 998 }) {
        for (int cf = 1; cf <= 4; cf++) {
            if (start == 0)
                iter.seek(new Range(), newColFamByteSequence(formatString("cf_", cf)), true);
            else
                iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(formatString("cf_", cf)), true);
            for (int i = start; i < 1000; i++) {
                assertTrue(iter.hasTop());
                assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey());
                assertEquals(newValue(i + ""), iter.getTopValue());
                iter.next();
            }
            assertFalse(iter.hasTop());
        }
        if (start == 0)
            iter.seek(new Range(), newColFamByteSequence(), false);
        else
            iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(), false);
        for (int i = start; i < 1000; i++) {
            for (int cf = 1; cf <= 4; cf++) {
                assertTrue(iter.hasTop());
                assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey());
                assertEquals(newValue(i + ""), iter.getTopValue());
                iter.next();
            }
        }
        assertFalse(iter.hasTop());
    }
    reader.close();
}

Example 2 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class LargeRowFilterTest method setupIterator.

private LargeRowFilter setupIterator(TreeMap<Key, Value> testData, int maxColumns, IteratorScope scope) throws IOException {
    SortedMapIterator smi = new SortedMapIterator(testData);
    LargeRowFilter lrfi = new LargeRowFilter();
    IteratorSetting is = new IteratorSetting(1, LargeRowFilter.class);
    LargeRowFilter.setMaxColumns(is, maxColumns);
    lrfi.init(new ColumnFamilySkippingIterator(smi), is.getOptions(), new RowDeletingIteratorTest.TestIE(scope, false));
    return lrfi;
}

Example 3 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class Compactor method compactLocalityGroup.

private void compactLocalityGroup(String lgName, Set<ByteSequence> columnFamilies, boolean inclusive, FileSKVWriter mfw, CompactionStats majCStats) throws IOException, CompactionCanceledException {
    ArrayList<FileSKVIterator> readers = new ArrayList<>(filesToCompact.size());
    Span span = Trace.start("compact");
    try {
        long entriesCompacted = 0;
        List<SortedKeyValueIterator<Key, Value>> iters = openMapDataFiles(lgName, readers);
        if (imm != null) {
            iters.add(imm.compactionIterator());
        }
        CountingIterator citr = new CountingIterator(new MultiIterator(iters, extent.toDataRange()), entriesRead);
        DeletingIterator delIter = new DeletingIterator(citr, propogateDeletes);
        ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(delIter);
        // if(env.getIteratorScope() )
        TabletIteratorEnvironment iterEnv;
        if (env.getIteratorScope() == IteratorScope.majc)
            iterEnv = new TabletIteratorEnvironment(IteratorScope.majc, !propogateDeletes, acuTableConf);
        else if (env.getIteratorScope() == IteratorScope.minc)
            iterEnv = new TabletIteratorEnvironment(IteratorScope.minc, acuTableConf);
        else
            throw new IllegalArgumentException();
        SortedKeyValueIterator<Key, Value> itr = iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(env.getIteratorScope(), cfsi, extent, acuTableConf, iterators, iterEnv));
        itr.seek(extent.toDataRange(), columnFamilies, inclusive);
        if (!inclusive) {
            mfw.startDefaultLocalityGroup();
        } else {
            mfw.startNewLocalityGroup(lgName, columnFamilies);
        }
        Span write = Trace.start("write");
        try {
            while (itr.hasTop() && env.isCompactionEnabled()) {
                mfw.append(itr.getTopKey(), itr.getTopValue());
                itr.next();
                entriesCompacted++;
                if (entriesCompacted % 1024 == 0) {
                    // Periodically update stats, do not want to do this too often since its volatile
                    entriesWritten.addAndGet(1024);
                }
            }
            if (itr.hasTop() && !env.isCompactionEnabled()) {
                // cancel major compaction operation
                try {
                    try {
                        mfw.close();
                    } catch (IOException e) {
                        log.error("{}", e.getMessage(), e);
                    }
                    fs.deleteRecursively(outputFile.path());
                } catch (Exception e) {
                    log.warn("Failed to delete Canceled compaction output file {}", outputFile, e);
                }
                throw new CompactionCanceledException();
            }
        } finally {
            CompactionStats lgMajcStats = new CompactionStats(citr.getCount(), entriesCompacted);
            majCStats.add(lgMajcStats);
            write.stop();
        }
    } finally {
        // close sequence files opened
        for (FileSKVIterator reader : readers) {
            try {
                reader.close();
            } catch (Throwable e) {
                log.warn("Failed to close map file", e);
            }
        }
        span.stop();
    }
}

Example 4 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class MockScannerBase method createFilter.

public SortedKeyValueIterator<Key, Value> createFilter(SortedKeyValueIterator<Key, Value> inner) throws IOException {
    byte[] defaultLabels = {};
    inner = new ColumnFamilySkippingIterator(new DeletingIterator(inner, false));
    SortedKeyValueIterator<Key, Value> cqf = ColumnQualifierFilter.wrap(inner, new HashSet<>(fetchedColumns));
    SortedKeyValueIterator<Key, Value> vf = VisibilityFilter.wrap(cqf, auths, defaultLabels);
    AccumuloConfiguration conf = new MockConfiguration(table.settings);
    MockIteratorEnvironment iterEnv = new MockIteratorEnvironment(auths);
    SortedKeyValueIterator<Key, Value> result = iterEnv.getTopLevelIterator(IteratorUtil.loadIterators(IteratorScope.scan, vf, null, conf, serverSideIteratorList, serverSideIteratorOptions, iterEnv, false));
    return result;
}

Example 5 with ColumnFamilySkippingIterator

use of org.apache.accumulo.core.iterators.system.ColumnFamilySkippingIterator in project accumulo by apache.

the class InMemoryMapTest method testBug1.

@Test
public void testBug1() throws Exception {
    InMemoryMap imm = newInMemoryMap(false, tempFolder.newFolder().getAbsolutePath());
    for (int i = 0; i < 20; i++) {
        mutate(imm, "r1", "foo:cq" + i, 3, "bar" + i);
    }
    for (int i = 0; i < 20; i++) {
        mutate(imm, "r2", "foo:cq" + i, 3, "bar" + i);
    }
    MemoryIterator ski1 = imm.skvIterator(null);
    ColumnFamilySkippingIterator cfsi = new ColumnFamilySkippingIterator(ski1);
    imm.delete(0);
    ArrayList<ByteSequence> columns = new ArrayList<>();
    columns.add(new ArrayByteSequence("bar"));
    // this seek resulted in an infinite loop before a bug was fixed
    cfsi.seek(new Range("r1"), columns, true);
    assertFalse(cfsi.hasTop());
    ski1.close();
}