use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.
the class FilterUtil method getForwardDictionaryCache.
/**
* @param tableIdentifier
* @param carbonDimension
* @return
*/
public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier, CarbonDimension carbonDimension) throws IOException {
DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(), carbonDimension.getColumnIdentifier(), carbonDimension.getDataType());
CacheProvider cacheProvider = CacheProvider.getInstance();
Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, tableIdentifier.getStorePath());
// get the forward dictionary object
return forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
}
use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.
the class IncrementalColumnDictionaryGenerator method writeDictionaryData.
@Override
public void writeDictionaryData(String tableUniqueName) throws IOException {
// initialize params
CarbonMetadata metadata = CarbonMetadata.getInstance();
CarbonTable carbonTable = metadata.getCarbonTable(tableUniqueName);
CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier();
ColumnIdentifier columnIdentifier = dimension.getColumnIdentifier();
String storePath = carbonTable.getStorePath();
DictionaryService dictionaryService = CarbonCommonFactory.getDictionaryService();
// create dictionary cache from dictionary File
DictionaryColumnUniqueIdentifier identifier = new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, columnIdentifier.getDataType());
Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(storePath, identifier);
Dictionary dictionary = null;
long t1 = System.currentTimeMillis();
if (isDictExists) {
Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, storePath);
dictionary = dictCache.get(identifier);
}
long dictCacheTime = System.currentTimeMillis() - t1;
long t2 = System.currentTimeMillis();
// write dictionary
CarbonDictionaryWriter dictionaryWriter = null;
dictionaryWriter = dictionaryService.getDictionaryWriter(tableIdentifier, columnIdentifier, storePath);
List<String> distinctValues = writeDictionary(dictionaryWriter, isDictExists);
long dictWriteTime = System.currentTimeMillis() - t2;
long t3 = System.currentTimeMillis();
// write sort index
if (distinctValues.size() > 0) {
writeSortIndex(distinctValues, dictionary, dictionaryService, tableIdentifier, columnIdentifier, storePath);
}
long sortIndexWriteTime = System.currentTimeMillis() - t3;
// update Meta Data
updateMetaData(dictionaryWriter);
LOGGER.audit("\n columnName: " + dimension.getColName() + "\n columnId: " + dimension.getColumnId() + "\n new distinct values count: " + distinctValues.size() + "\n create dictionary cache: " + dictCacheTime + "\n sort list, distinct and write: " + dictWriteTime + "\n write sort info: " + sortIndexWriteTime);
}
use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.
the class RowLevelFilterExecuterImpl method getFilterActualValueFromDictionaryValue.
/**
* Read the actual filter member by passing the dictionary value from
* the forward dictionary cache which which holds column wise cache
*
* @param dimColumnEvaluatorInfo
* @param dictionaryValue
* @return
* @throws IOException
*/
private String getFilterActualValueFromDictionaryValue(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) throws IOException {
String memberString;
Dictionary forwardDictionary = FilterUtil.getForwardDictionaryCache(tableIdentifier, dimColumnEvaluatorInfo.getDimension());
memberString = forwardDictionary.getDictionaryValueForKey(dictionaryValue);
if (null != memberString) {
if (memberString.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
memberString = null;
}
}
return memberString;
}
use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.
the class CacheProvider method createDictionaryCacheForGivenType.
/**
* This method will create the cache for given cache type
*
* @param cacheType type of cache
* @param carbonStorePath store path
*/
private void createDictionaryCacheForGivenType(CacheType cacheType, String carbonStorePath) {
Cache cacheObject = null;
if (cacheType.equals(CacheType.REVERSE_DICTIONARY)) {
cacheObject = new ReverseDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath, carbonLRUCache);
} else if (cacheType.equals(CacheType.FORWARD_DICTIONARY)) {
cacheObject = new ForwardDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath, carbonLRUCache);
} else if (cacheType.equals(cacheType.EXECUTOR_BTREE)) {
cacheObject = new BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex>(carbonStorePath, carbonLRUCache);
} else if (cacheType.equals(cacheType.DRIVER_BTREE)) {
cacheObject = new SegmentTaskIndexStore(carbonStorePath, carbonLRUCache);
}
cacheTypeToCacheMap.put(cacheType, cacheObject);
}
use of org.apache.carbondata.core.cache.dictionary.Dictionary in project carbondata by apache.
the class CarbonDictionarySortInfoPreparatorTest method testGetDictionarySortInfoDictionaryNullCase.
/**
* Tests getDictionarySortInfo when dictionary is null
*/
@Test
public void testGetDictionarySortInfoDictionaryNullCase() {
List<String> newDistinctValues = new ArrayList<>();
newDistinctValues.add("abc");
newDistinctValues.add("xyz");
Dictionary dictionary = null;
CarbonDictionarySortInfo carbonDictionarySortInfo = carbonDictionarySortInfoPreparator.getDictionarySortInfo(newDistinctValues, dictionary, DataType.ARRAY);
int expectedGetSortIndexValue = 1;
int expectedGetSortInvertedIndexLength = 2;
int actualGetSortIndexValue = carbonDictionarySortInfo.getSortIndex().get(0);
int actualGetSortInvertedIndexLength = carbonDictionarySortInfo.getSortIndexInverted().size();
assertEquals(actualGetSortIndexValue, expectedGetSortIndexValue);
assertEquals(actualGetSortInvertedIndexLength, expectedGetSortInvertedIndexLength);
}
Aggregations