Examples with DefaultConfiguration org.apache.accumulo.core.conf.DefaultConfiguration used on opensource projects

Example 6 with DefaultConfiguration

use of org.apache.accumulo.core.conf.DefaultConfiguration in project accumulo by apache.

the class TestLruBlockCache method testScanResistance.

// test scan resistance
public void testScanResistance() throws Exception {
    long maxSize = 100000;
    long blockSize = calculateBlockSize(maxSize, 10);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.66f).acceptableFactor(0.99f).singleFactor(0.33f).multiFactor(0.33f).memoryFactor(0.34f).buildMap().forEach(cc::set);
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] singleBlocks = generateFixedBlocks(20, blockSize, "single");
    Block[] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
    // Add 5 multi blocks
    for (Block block : multiBlocks) {
        cache.cacheBlock(block.blockName, block.buf);
        cache.getBlock(block.blockName);
    }
    // Add 5 single blocks
    for (int i = 0; i < 5; i++) {
        cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i].buf);
    }
    // An eviction ran
    assertEquals(1, cache.getEvictionCount());
    // To drop down to 2/3 capacity, we'll need to evict 4 blocks
    assertEquals(4, cache.getEvictedCount());
    // Should have been taken off equally from single and multi
    assertEquals(null, cache.getBlock(singleBlocks[0].blockName));
    assertEquals(null, cache.getBlock(singleBlocks[1].blockName));
    assertEquals(null, cache.getBlock(multiBlocks[0].blockName));
    assertEquals(null, cache.getBlock(multiBlocks[1].blockName));
    for (int i = 5; i < 18; i++) {
        cache.cacheBlock(singleBlocks[i].blockName, singleBlocks[i].buf);
    }
    // 4 total evictions, 16 total evicted
    assertEquals(4, cache.getEvictionCount());
    assertEquals(16, cache.getEvictedCount());
    // Should now have 7 total blocks
    assertEquals(7, cache.size());
    manager.stop();
}

Example 7 with DefaultConfiguration

use of org.apache.accumulo.core.conf.DefaultConfiguration in project accumulo by apache.

the class TestLruBlockCache method testBackgroundEvictionThread.

public void testBackgroundEvictionThread() throws Exception {
    long maxSize = 100000;
    // room for 9, will evict
    long blockSize = calculateBlockSizeDefault(maxSize, 9);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] blocks = generateFixedBlocks(10, blockSize, "block");
    // Add all the blocks
    for (Block block : blocks) {
        cache.cacheBlock(block.blockName, block.buf);
    }
    // Let the eviction run
    int n = 0;
    while (cache.getEvictionCount() == 0) {
        Thread.sleep(1000);
        assertTrue(n++ < 1);
    }
    // A single eviction run should have occurred
    assertEquals(cache.getEvictionCount(), 1);
    manager.stop();
}

Example 8 with DefaultConfiguration

use of org.apache.accumulo.core.conf.DefaultConfiguration in project accumulo by apache.

the class TestLruBlockCache method testCacheEvictionTwoPriorities.

public void testCacheEvictionTwoPriorities() throws Exception {
    long maxSize = 100000;
    long blockSize = calculateBlockSizeDefault(maxSize, 10);
    DefaultConfiguration dc = DefaultConfiguration.getInstance();
    ConfigurationCopy cc = new ConfigurationCopy(dc);
    cc.set(Property.TSERV_CACHE_MANAGER_IMPL, LruBlockCacheManager.class.getName());
    BlockCacheManager manager = BlockCacheManagerFactory.getInstance(cc);
    cc.set(Property.TSERV_DEFAULT_BLOCKSIZE, Long.toString(blockSize));
    cc.set(Property.TSERV_INDEXCACHE_SIZE, Long.toString(maxSize));
    LruBlockCacheConfiguration.builder(CacheType.INDEX).useEvictionThread(false).minFactor(0.98f).acceptableFactor(0.99f).singleFactor(0.25f).multiFactor(0.50f).memoryFactor(0.25f).buildMap().forEach(cc::set);
    manager.start(new BlockCacheConfiguration(cc));
    LruBlockCache cache = (LruBlockCache) manager.getBlockCache(CacheType.INDEX);
    Block[] singleBlocks = generateFixedBlocks(5, 10000, "single");
    Block[] multiBlocks = generateFixedBlocks(5, 10000, "multi");
    long expectedCacheSize = cache.heapSize();
    // Add and get the multi blocks
    for (Block block : multiBlocks) {
        cache.cacheBlock(block.blockName, block.buf);
        expectedCacheSize += block.heapSize();
        assertTrue(Arrays.equals(cache.getBlock(block.blockName).getBuffer(), block.buf));
    }
    // Add the single blocks (no get)
    for (Block block : singleBlocks) {
        cache.cacheBlock(block.blockName, block.buf);
        expectedCacheSize += block.heapSize();
    }
    // A single eviction run should have occurred
    assertEquals(cache.getEvictionCount(), 1);
    // We expect two entries evicted
    assertEquals(cache.getEvictedCount(), 2);
    // Our expected size overruns acceptable limit
    assertTrue(expectedCacheSize > (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
    // But the cache did not grow beyond max
    assertTrue(cache.heapSize() <= maxSize);
    // And is now below the acceptable limit
    assertTrue(cache.heapSize() <= (maxSize * LruBlockCacheConfiguration.DEFAULT_ACCEPTABLE_FACTOR));
    // We expect fairness across the two priorities.
    // This test makes multi go barely over its limit, in-memory
    // empty, and the rest in single. Two single evictions and
    // one multi eviction expected.
    assertTrue(cache.getBlock(singleBlocks[0].blockName) == null);
    assertTrue(cache.getBlock(multiBlocks[0].blockName) == null);
    // And all others to be cached
    for (int i = 1; i < 4; i++) {
        assertTrue(Arrays.equals(cache.getBlock(singleBlocks[i].blockName).getBuffer(), singleBlocks[i].buf));
        assertTrue(Arrays.equals(cache.getBlock(multiBlocks[i].blockName).getBuffer(), multiBlocks[i].buf));
    }
    manager.stop();
}

Example 9 with DefaultConfiguration

use of org.apache.accumulo.core.conf.DefaultConfiguration in project accumulo by apache.

the class AccumuloFileOutputFormatIT method handleWriteTests.

private void handleWriteTests(boolean content) throws Exception {
    File f = folder.newFile(testName.getMethodName());
    assertTrue(f.delete());
    MRTester.main(new String[] { content ? TEST_TABLE : EMPTY_TABLE, f.getAbsolutePath() });
    assertTrue(f.exists());
    File[] files = f.listFiles(new FileFilter() {

        @Override
        public boolean accept(File file) {
            return file.getName().startsWith("part-m-");
        }
    });
    assertNotNull(files);
    if (content) {
        assertEquals(1, files.length);
        assertTrue(files[0].exists());
        Configuration conf = CachedConfiguration.getInstance();
        DefaultConfiguration acuconf = DefaultConfiguration.getInstance();
        FileSKVIterator sample = RFileOperations.getInstance().newReaderBuilder().forFile(files[0].toString(), FileSystem.get(conf), conf).withTableConfiguration(acuconf).build().getSample(new SamplerConfigurationImpl(SAMPLER_CONFIG));
        assertNotNull(sample);
    } else {
        assertEquals(0, files.length);
    }
}

Example 10 with DefaultConfiguration

use of org.apache.accumulo.core.conf.DefaultConfiguration in project accumulo by apache.

the class WriteExportFiles method exportConfig.

private static void exportConfig(AccumuloServerContext context, Table.ID tableID, ZipOutputStream zipOut, DataOutputStream dataOut) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, IOException {
    Connector conn = context.getConnector();
    DefaultConfiguration defaultConfig = DefaultConfiguration.getInstance();
    Map<String, String> siteConfig = conn.instanceOperations().getSiteConfiguration();
    Map<String, String> systemConfig = conn.instanceOperations().getSystemConfiguration();
    TableConfiguration tableConfig = context.getServerConfigurationFactory().getTableConfiguration(tableID);
    OutputStreamWriter osw = new OutputStreamWriter(dataOut, UTF_8);
    // only put props that are different than defaults and higher level configurations
    zipOut.putNextEntry(new ZipEntry(Constants.EXPORT_TABLE_CONFIG_FILE));
    for (Entry<String, String> prop : tableConfig) {
        if (prop.getKey().startsWith(Property.TABLE_PREFIX.getKey())) {
            Property key = Property.getPropertyByKey(prop.getKey());
            if (key == null || !defaultConfig.get(key).equals(prop.getValue())) {
                if (!prop.getValue().equals(siteConfig.get(prop.getKey())) && !prop.getValue().equals(systemConfig.get(prop.getKey()))) {
                    osw.append(prop.getKey() + "=" + prop.getValue() + "\n");
                }
            }
        }
    }
    osw.flush();
}