Examples with SegmentTaskIndexStore org.apache.carbondata.core.datastore.SegmentTaskIndexStore used on opensource projects

Example 1 with SegmentTaskIndexStore

use of org.apache.carbondata.core.datastore.SegmentTaskIndexStore in project carbondata by apache.

the class CacheProvider method createDictionaryCacheForGivenType.

/**
   * This method will create the cache for given cache type
   *
   * @param cacheType       type of cache
   * @param carbonStorePath store path
   */
private void createDictionaryCacheForGivenType(CacheType cacheType, String carbonStorePath) {
    Cache cacheObject = null;
    if (cacheType.equals(CacheType.REVERSE_DICTIONARY)) {
        cacheObject = new ReverseDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath, carbonLRUCache);
    } else if (cacheType.equals(CacheType.FORWARD_DICTIONARY)) {
        cacheObject = new ForwardDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath, carbonLRUCache);
    } else if (cacheType.equals(cacheType.EXECUTOR_BTREE)) {
        cacheObject = new BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex>(carbonStorePath, carbonLRUCache);
    } else if (cacheType.equals(cacheType.DRIVER_BTREE)) {
        cacheObject = new SegmentTaskIndexStore(carbonStorePath, carbonLRUCache);
    }
    cacheTypeToCacheMap.put(cacheType, cacheObject);
}

Example 2 with SegmentTaskIndexStore

use of org.apache.carbondata.core.datastore.SegmentTaskIndexStore in project carbondata by apache.

the class CacheProviderTest method driverExecutorCacheConfTest.

/**
 * to test the driver and executor lru memory configuration
 *
 * @throws IOException
 * @throws NoSuchFieldException
 * @throws IllegalAccessException
 */
@Test
public void driverExecutorCacheConfTest() throws IOException, NoSuchFieldException, IllegalAccessException {
    // get cache provider instance
    CacheProvider cacheProvider = CacheProvider.getInstance();
    cacheProvider.dropAllCache();
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true");
    Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexStore> driverCache = cacheProvider.createCache(CacheType.DRIVER_BTREE);
    Field carbonLRUCacheField = SegmentTaskIndexStore.class.getDeclaredField("lruCache");
    carbonLRUCacheField.setAccessible(true);
    CarbonLRUCache carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(driverCache);
    Field lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize");
    lruCacheMemorySizeField.setAccessible(true);
    long lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache);
    String driverCacheSize = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE);
    assertEquals(1024 * 1024 * Integer.parseInt(driverCacheSize), lruCacheMemorySize);
    // drop cache
    cacheProvider.dropAllCache();
    // validation test for the executor memory.
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "false");
    Cache<TableBlockUniqueIdentifier, BlockIndexStore> executorCache = cacheProvider.createCache(CacheType.EXECUTOR_BTREE);
    carbonLRUCacheField = BlockIndexStore.class.getSuperclass().getDeclaredField("lruCache");
    carbonLRUCacheField.setAccessible(true);
    carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(executorCache);
    lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize");
    lruCacheMemorySizeField.setAccessible(true);
    lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache);
    String executorCacheSize = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE);
    assertEquals(1024 * 1024 * Integer.parseInt(executorCacheSize), lruCacheMemorySize);
    cacheProvider.dropAllCache();
}

Example 3 with SegmentTaskIndexStore

use of org.apache.carbondata.core.datastore.SegmentTaskIndexStore in project carbondata by apache.

the class CacheProvider method createDictionaryCacheForGivenType.

/**
 * This method will create the cache for given cache type
 *
 * @param cacheType       type of cache
 */
private void createDictionaryCacheForGivenType(CacheType cacheType) {
    Cache cacheObject = null;
    if (cacheType.equals(CacheType.REVERSE_DICTIONARY)) {
        cacheObject = new ReverseDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonLRUCache);
    } else if (cacheType.equals(CacheType.FORWARD_DICTIONARY)) {
        cacheObject = new ForwardDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonLRUCache);
    } else if (cacheType.equals(cacheType.EXECUTOR_BTREE)) {
        cacheObject = new BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex>(carbonLRUCache);
    } else if (cacheType.equals(cacheType.DRIVER_BTREE)) {
        cacheObject = new SegmentTaskIndexStore(carbonLRUCache);
    } else if (cacheType.equals(cacheType.DRIVER_BLOCKLET_DATAMAP)) {
        cacheObject = new BlockletDataMapIndexStore(carbonLRUCache);
    }
    cacheTypeToCacheMap.put(cacheType, cacheObject);
}