Examples with Dictionary org.apache.carbondata.core.cache.dictionary.Dictionary used on opensource projects

Example 1 with Dictionary

use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.

the class FilterUtil method getForwardDictionaryCache.

/**
   * @param tableIdentifier
   * @param carbonDimension
   * @return
   */
public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier, CarbonDimension carbonDimension) throws IOException {
    DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(), carbonDimension.getColumnIdentifier(), carbonDimension.getDataType());
    CacheProvider cacheProvider = CacheProvider.getInstance();
    Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, tableIdentifier.getStorePath());
    // get the forward dictionary object
    return forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
}

Example 2 with Dictionary

use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.

the class IncrementalColumnDictionaryGenerator method writeDictionaryData.

@Override
public void writeDictionaryData(String tableUniqueName) throws IOException {
    // initialize params
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    CarbonTable carbonTable = metadata.getCarbonTable(tableUniqueName);
    CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier();
    ColumnIdentifier columnIdentifier = dimension.getColumnIdentifier();
    String storePath = carbonTable.getStorePath();
    DictionaryService dictionaryService = CarbonCommonFactory.getDictionaryService();
    // create dictionary cache from dictionary File
    DictionaryColumnUniqueIdentifier identifier = new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, columnIdentifier.getDataType());
    Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(storePath, identifier);
    Dictionary dictionary = null;
    long t1 = System.currentTimeMillis();
    if (isDictExists) {
        Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, storePath);
        dictionary = dictCache.get(identifier);
    }
    long dictCacheTime = System.currentTimeMillis() - t1;
    long t2 = System.currentTimeMillis();
    // write dictionary
    CarbonDictionaryWriter dictionaryWriter = null;
    dictionaryWriter = dictionaryService.getDictionaryWriter(tableIdentifier, columnIdentifier, storePath);
    List<String> distinctValues = writeDictionary(dictionaryWriter, isDictExists);
    long dictWriteTime = System.currentTimeMillis() - t2;
    long t3 = System.currentTimeMillis();
    // write sort index
    if (distinctValues.size() > 0) {
        writeSortIndex(distinctValues, dictionary, dictionaryService, tableIdentifier, columnIdentifier, storePath);
    }
    long sortIndexWriteTime = System.currentTimeMillis() - t3;
    // update Meta Data
    updateMetaData(dictionaryWriter);
    LOGGER.audit("\n columnName: " + dimension.getColName() + "\n columnId: " + dimension.getColumnId() + "\n new distinct values count: " + distinctValues.size() + "\n create dictionary cache: " + dictCacheTime + "\n sort list, distinct and write: " + dictWriteTime + "\n write sort info: " + sortIndexWriteTime);
}

Example 3 with Dictionary

use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.

the class RowLevelFilterExecuterImpl method getFilterActualValueFromDictionaryValue.

/**
   * Read the actual filter member by passing the dictionary value from
   * the forward dictionary cache which which holds column wise cache
   *
   * @param dimColumnEvaluatorInfo
   * @param dictionaryValue
   * @return
   * @throws IOException
   */
private String getFilterActualValueFromDictionaryValue(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) throws IOException {
    String memberString;
    Dictionary forwardDictionary = FilterUtil.getForwardDictionaryCache(tableIdentifier, dimColumnEvaluatorInfo.getDimension());
    memberString = forwardDictionary.getDictionaryValueForKey(dictionaryValue);
    if (null != memberString) {
        if (memberString.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
            memberString = null;
        }
    }
    return memberString;
}

Example 4 with Dictionary

use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.

the class CacheProvider method createDictionaryCacheForGivenType.

/**
   * This method will create the cache for given cache type
   *
   * @param cacheType       type of cache
   * @param carbonStorePath store path
   */
private void createDictionaryCacheForGivenType(CacheType cacheType, String carbonStorePath) {
    Cache cacheObject = null;
    if (cacheType.equals(CacheType.REVERSE_DICTIONARY)) {
        cacheObject = new ReverseDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath, carbonLRUCache);
    } else if (cacheType.equals(CacheType.FORWARD_DICTIONARY)) {
        cacheObject = new ForwardDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath, carbonLRUCache);
    } else if (cacheType.equals(cacheType.EXECUTOR_BTREE)) {
        cacheObject = new BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex>(carbonStorePath, carbonLRUCache);
    } else if (cacheType.equals(cacheType.DRIVER_BTREE)) {
        cacheObject = new SegmentTaskIndexStore(carbonStorePath, carbonLRUCache);
    }
    cacheTypeToCacheMap.put(cacheType, cacheObject);
}

Example 5 with Dictionary

use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.

the class CarbonDictionarySortInfoPreparatorTest method testGetDictionarySortInfoDictionaryNullCase.

/**
   * Tests getDictionarySortInfo when dictionary is null
   */
@Test
public void testGetDictionarySortInfoDictionaryNullCase() {
    List<String> newDistinctValues = new ArrayList<>();
    newDistinctValues.add("abc");
    newDistinctValues.add("xyz");
    Dictionary dictionary = null;
    CarbonDictionarySortInfo carbonDictionarySortInfo = carbonDictionarySortInfoPreparator.getDictionarySortInfo(newDistinctValues, dictionary, DataType.ARRAY);
    int expectedGetSortIndexValue = 1;
    int expectedGetSortInvertedIndexLength = 2;
    int actualGetSortIndexValue = carbonDictionarySortInfo.getSortIndex().get(0);
    int actualGetSortInvertedIndexLength = carbonDictionarySortInfo.getSortIndexInverted().size();
    assertEquals(actualGetSortIndexValue, expectedGetSortIndexValue);
    assertEquals(actualGetSortInvertedIndexLength, expectedGetSortInvertedIndexLength);
}