Examples with TableBlockUniqueIdentifier org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier used on opensource projects

Example 6 with TableBlockUniqueIdentifier

use of org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier in project carbondata by apache.

the class BlockIndexStoreTest method testloadAndGetTaskIdToSegmentsMapForSameBlockLoadedConcurrently.

@Test
public void testloadAndGetTaskIdToSegmentsMapForSameBlockLoadedConcurrently() throws IOException {
    String canonicalPath = new File(this.getClass().getResource("/").getPath() + "/../../").getCanonicalPath();
    File file = getPartFile();
    TableBlockInfo info = new TableBlockInfo(file.getAbsolutePath(), 0, "0", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info1 = new TableBlockInfo(file.getAbsolutePath(), 0, "0", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info2 = new TableBlockInfo(file.getAbsolutePath(), 0, "1", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info3 = new TableBlockInfo(file.getAbsolutePath(), 0, "1", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info4 = new TableBlockInfo(file.getAbsolutePath(), 0, "1", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "t3", "1");
    AbsoluteTableIdentifier absoluteTableIdentifier = new AbsoluteTableIdentifier("/src/test/resources", carbonTableIdentifier);
    ExecutorService executor = Executors.newFixedThreadPool(3);
    executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info, info1 }), absoluteTableIdentifier));
    executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info2, info3, info4 }), absoluteTableIdentifier));
    executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info, info1 }), absoluteTableIdentifier));
    executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info2, info3, info4 }), absoluteTableIdentifier));
    executor.shutdown();
    try {
        executor.awaitTermination(1, TimeUnit.DAYS);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    List<TableBlockInfo> tableBlockInfos = Arrays.asList(new TableBlockInfo[] { info, info1, info2, info3, info4 });
    try {
        List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers = getTableBlockUniqueIdentifierList(tableBlockInfos, absoluteTableIdentifier);
        List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockUniqueIdentifiers);
        assertTrue(loadAndGetBlocks.size() == 5);
    } catch (Exception e) {
        assertTrue(false);
    }
    List<String> segmentIds = new ArrayList<>();
    for (TableBlockInfo tableBlockInfo : tableBlockInfos) {
        segmentIds.add(tableBlockInfo.getSegmentId());
    }
    cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
}

Example 7 with TableBlockUniqueIdentifier

use of org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier in project carbondata by apache.

the class BlockIndexStoreTest method testloadAndGetTaskIdToSegmentsMapForDifferentSegmentLoadedConcurrently.

@Test
public void testloadAndGetTaskIdToSegmentsMapForDifferentSegmentLoadedConcurrently() throws IOException {
    String canonicalPath = new File(this.getClass().getResource("/").getPath() + "/../../").getCanonicalPath();
    File file = getPartFile();
    TableBlockInfo info = new TableBlockInfo(file.getAbsolutePath(), 0, "0", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info1 = new TableBlockInfo(file.getAbsolutePath(), 0, "0", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info2 = new TableBlockInfo(file.getAbsolutePath(), 0, "1", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info3 = new TableBlockInfo(file.getAbsolutePath(), 0, "1", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info4 = new TableBlockInfo(file.getAbsolutePath(), 0, "1", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info5 = new TableBlockInfo(file.getAbsolutePath(), 0, "2", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info6 = new TableBlockInfo(file.getAbsolutePath(), 0, "2", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    TableBlockInfo info7 = new TableBlockInfo(file.getAbsolutePath(), 0, "3", new String[] { "loclhost" }, file.length(), ColumnarFormatVersion.V1);
    CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier(CarbonCommonConstants.DATABASE_DEFAULT_NAME, "t3", "1");
    AbsoluteTableIdentifier absoluteTableIdentifier = new AbsoluteTableIdentifier("/src/test/resources", carbonTableIdentifier);
    ExecutorService executor = Executors.newFixedThreadPool(3);
    executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info, info1 }), absoluteTableIdentifier));
    executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info2, info3, info4 }), absoluteTableIdentifier));
    executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info5, info6 }), absoluteTableIdentifier));
    executor.submit(new BlockLoaderThread(Arrays.asList(new TableBlockInfo[] { info7 }), absoluteTableIdentifier));
    executor.shutdown();
    try {
        executor.awaitTermination(1, TimeUnit.DAYS);
    } catch (InterruptedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    List<TableBlockInfo> tableBlockInfos = Arrays.asList(new TableBlockInfo[] { info, info1, info2, info3, info4, info5, info6, info7 });
    try {
        List<TableBlockUniqueIdentifier> blockUniqueIdentifierList = getTableBlockUniqueIdentifierList(tableBlockInfos, absoluteTableIdentifier);
        List<AbstractIndex> loadAndGetBlocks = cache.getAll(blockUniqueIdentifierList);
        assertTrue(loadAndGetBlocks.size() == 8);
    } catch (Exception e) {
        assertTrue(false);
    }
    List<String> segmentIds = new ArrayList<>();
    for (TableBlockInfo tableBlockInfo : tableBlockInfos) {
        segmentIds.add(tableBlockInfo.getSegmentId());
    }
    cache.removeTableBlocks(segmentIds, absoluteTableIdentifier);
}

Example 8 with TableBlockUniqueIdentifier

use of org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier in project carbondata by apache.

the class CacheProviderTest method driverExecutorCacheConfTest.

/**
 * to test the driver and executor lru memory configuration
 *
 * @throws IOException
 * @throws NoSuchFieldException
 * @throws IllegalAccessException
 */
@Test
public void driverExecutorCacheConfTest() throws IOException, NoSuchFieldException, IllegalAccessException {
    // get cache provider instance
    CacheProvider cacheProvider = CacheProvider.getInstance();
    cacheProvider.dropAllCache();
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true");
    Cache<TableSegmentUniqueIdentifier, SegmentTaskIndexStore> driverCache = cacheProvider.createCache(CacheType.DRIVER_BTREE);
    Field carbonLRUCacheField = SegmentTaskIndexStore.class.getDeclaredField("lruCache");
    carbonLRUCacheField.setAccessible(true);
    CarbonLRUCache carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(driverCache);
    Field lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize");
    lruCacheMemorySizeField.setAccessible(true);
    long lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache);
    String driverCacheSize = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE);
    assertEquals(1024 * 1024 * Integer.parseInt(driverCacheSize), lruCacheMemorySize);
    // drop cache
    cacheProvider.dropAllCache();
    // validation test for the executor memory.
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "false");
    Cache<TableBlockUniqueIdentifier, BlockIndexStore> executorCache = cacheProvider.createCache(CacheType.EXECUTOR_BTREE);
    carbonLRUCacheField = BlockIndexStore.class.getSuperclass().getDeclaredField("lruCache");
    carbonLRUCacheField.setAccessible(true);
    carbonLRUCache = (CarbonLRUCache) carbonLRUCacheField.get(executorCache);
    lruCacheMemorySizeField = CarbonLRUCache.class.getDeclaredField("lruCacheMemorySize");
    lruCacheMemorySizeField.setAccessible(true);
    lruCacheMemorySize = (long) lruCacheMemorySizeField.get(carbonLRUCache);
    String executorCacheSize = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE);
    assertEquals(1024 * 1024 * Integer.parseInt(executorCacheSize), lruCacheMemorySize);
    cacheProvider.dropAllCache();
}

Example 9 with TableBlockUniqueIdentifier

use of org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier in project carbondata by apache.

the class CacheProvider method createDictionaryCacheForGivenType.

/**
 * This method will create the cache for given cache type
 *
 * @param cacheType       type of cache
 */
private void createDictionaryCacheForGivenType(CacheType cacheType) {
    Cache cacheObject = null;
    if (cacheType.equals(CacheType.REVERSE_DICTIONARY)) {
        cacheObject = new ReverseDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonLRUCache);
    } else if (cacheType.equals(CacheType.FORWARD_DICTIONARY)) {
        cacheObject = new ForwardDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonLRUCache);
    } else if (cacheType.equals(cacheType.EXECUTOR_BTREE)) {
        cacheObject = new BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex>(carbonLRUCache);
    } else if (cacheType.equals(cacheType.DRIVER_BTREE)) {
        cacheObject = new SegmentTaskIndexStore(carbonLRUCache);
    } else if (cacheType.equals(cacheType.DRIVER_BLOCKLET_DATAMAP)) {
        cacheObject = new BlockletDataMapIndexStore(carbonLRUCache);
    }
    cacheTypeToCacheMap.put(cacheType, cacheObject);
}