Examples with BasicCacheProvider org.apache.accumulo.core.file.blockfile.impl.BasicCacheProvider used on opensource projects

Example 1 with BasicCacheProvider

use of org.apache.accumulo.core.file.blockfile.impl.BasicCacheProvider in project accumulo by apache.

the class SummaryReader method load.

public static SummaryReader load(FileSystem fs, Configuration conf, SummarizerFactory factory, Path file, Predicate<SummarizerConfiguration> summarySelector, BlockCache summaryCache, BlockCache indexCache, Cache<String, Long> fileLenCache, CryptoService cryptoService) {
    CachableBlockFile.Reader bcReader = null;
    try {
        // the reason BCFile is used instead of RFile is to avoid reading in the RFile meta block when
        // only summary data is wanted.
        CompositeCache compositeCache = new CompositeCache(summaryCache, indexCache);
        CachableBuilder cb = new CachableBuilder().fsPath(fs, file).conf(conf).fileLen(fileLenCache).cacheProvider(new BasicCacheProvider(compositeCache, null)).cryptoService(cryptoService);
        bcReader = new CachableBlockFile.Reader(cb);
        return load(bcReader, summarySelector, factory);
    } catch (FileNotFoundException fne) {
        return getEmptyReader(factory);
    } catch (IOException e) {
        try {
            if (!fs.exists(file)) {
                return getEmptyReader(factory);
            }
        } catch (IOException e1) {
        }
        throw new UncheckedIOException(e);
    } finally {
        if (bcReader != null) {
            try {
                bcReader.close();
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        }
    }
}

Example 2 with BasicCacheProvider

use of org.apache.accumulo.core.file.blockfile.impl.BasicCacheProvider in project accumulo by apache.

the class RFileScanner method iterator.

@Override
public Iterator<Entry<Key, Value>> iterator() {
    try {
        RFileSource[] sources = opts.in.getSources();
        List<SortedKeyValueIterator<Key, Value>> readers = new ArrayList<>(sources.length);
        CacheProvider cacheProvider = new BasicCacheProvider(indexCache, dataCache);
        for (int i = 0; i < sources.length; i++) {
            // TODO may have been a bug with multiple files and caching in older version...
            FSDataInputStream inputStream = (FSDataInputStream) sources[i].getInputStream();
            CachableBuilder cb = new CachableBuilder().input(inputStream, "source-" + i).length(sources[i].getLength()).conf(opts.in.getConf()).cacheProvider(cacheProvider).cryptoService(cryptoService);
            readers.add(new RFile.Reader(cb));
        }
        if (getSamplerConfiguration() != null) {
            for (int i = 0; i < readers.size(); i++) {
                readers.set(i, ((Reader) readers.get(i)).getSample(new SamplerConfigurationImpl(getSamplerConfiguration())));
            }
        }
        SortedKeyValueIterator<Key, Value> iterator;
        if (opts.bounds != null) {
            iterator = new MultiIterator(readers, opts.bounds);
        } else {
            iterator = new MultiIterator(readers, false);
        }
        Set<ByteSequence> families = Collections.emptySet();
        if (opts.useSystemIterators) {
            SortedSet<Column> cols = this.getFetchedColumns();
            families = LocalityGroupUtil.families(cols);
            iterator = SystemIteratorUtil.setupSystemScanIterators(iterator, cols, getAuthorizations(), EMPTY_BYTES, tableConf);
        }
        try {
            if (opts.tableConfig != null && !opts.tableConfig.isEmpty()) {
                IterLoad il = IterConfigUtil.loadIterConf(IteratorScope.scan, serverSideIteratorList, serverSideIteratorOptions, tableConf);
                iterator = IterConfigUtil.loadIterators(iterator, il.iterEnv(new IterEnv()).useAccumuloClassLoader(true));
            } else {
                iterator = IterConfigUtil.loadIterators(iterator, new IterLoad().iters(serverSideIteratorList).iterOpts(serverSideIteratorOptions).iterEnv(new IterEnv()).useAccumuloClassLoader(false));
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        iterator.seek(getRange() == null ? EMPTY_RANGE : getRange(), families, !families.isEmpty());
        return new IteratorAdapter(iterator);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 3 with BasicCacheProvider

use of org.apache.accumulo.core.file.blockfile.impl.BasicCacheProvider in project accumulo by apache.

the class RFileTest method runVersionTest.

private void runVersionTest(int version, ConfigurationCopy aconf) throws Exception {
    InputStream in = this.getClass().getClassLoader().getResourceAsStream("org/apache/accumulo/core/file/rfile/ver_" + version + ".rf");
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    byte[] buf = new byte[1024];
    int read;
    while ((read = in.read(buf)) > 0) baos.write(buf, 0, read);
    byte[] data = baos.toByteArray();
    SeekableByteArrayInputStream bais = new SeekableByteArrayInputStream(data);
    FSDataInputStream in2 = new FSDataInputStream(bais);
    aconf.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    aconf.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(100000));
    aconf.set(Property.TSERV_DATACACHE_SIZE, Long.toString(100000000));
    aconf.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(100000000));
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(aconf);
    manager.start(new BlockCacheConfiguration(aconf));
    CachableBuilder cb = new CachableBuilder().input(in2, "cache-1").length(data.length).conf(hadoopConf).cryptoService(CryptoServiceFactory.newInstance(aconf, ClassloaderType.JAVA)).cacheProvider(new BasicCacheProvider(manager.getBlockCache(CacheType.INDEX), manager.getBlockCache(CacheType.DATA)));
    Reader reader = new RFile.Reader(cb);
    checkIndex(reader);
    ColumnFamilySkippingIterator iter = new ColumnFamilySkippingIterator(reader);
    for (int start : new int[] { 0, 10, 100, 998 }) {
        for (int cf = 1; cf <= 4; cf++) {
            if (start == 0)
                iter.seek(new Range(), newColFamByteSequence(formatString("cf_", cf)), true);
            else
                iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(formatString("cf_", cf)), true);
            for (int i = start; i < 1000; i++) {
                assertTrue(iter.hasTop());
                assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey());
                assertEquals(newValue(i + ""), iter.getTopValue());
                iter.next();
            }
            assertFalse(iter.hasTop());
        }
        if (start == 0)
            iter.seek(new Range(), newColFamByteSequence(), false);
        else
            iter.seek(new Range(formatString("r_", start), null), newColFamByteSequence(), false);
        for (int i = start; i < 1000; i++) {
            for (int cf = 1; cf <= 4; cf++) {
                assertTrue(iter.hasTop());
                assertEquals(newKey(formatString("r_", i), formatString("cf_", cf), formatString("cq_", 0), "", 1000 - i), iter.getTopKey());
                assertEquals(newValue(i + ""), iter.getTopValue());
                iter.next();
            }
        }
        assertFalse(iter.hasTop());
    }
    manager.stop();
    reader.close();
}