Examples with DictionaryColumnUniqueIdentifier org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier used on opensource projects

Example 1 with DictionaryColumnUniqueIdentifier

use of org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier in project carbondata by apache.

the class FilterUtil method getForwardDictionaryCache.

/**
   * @param tableIdentifier
   * @param carbonDimension
   * @return
   */
public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier, CarbonDimension carbonDimension) throws IOException {
    DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(), carbonDimension.getColumnIdentifier(), carbonDimension.getDataType());
    CacheProvider cacheProvider = CacheProvider.getInstance();
    Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, tableIdentifier.getStorePath());
    // get the forward dictionary object
    return forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
}

Example 2 with DictionaryColumnUniqueIdentifier

use of org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier in project carbondata by apache.

the class IncrementalColumnDictionaryGenerator method writeDictionaryData.

@Override
public void writeDictionaryData(String tableUniqueName) throws IOException {
    // initialize params
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    CarbonTable carbonTable = metadata.getCarbonTable(tableUniqueName);
    CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier();
    ColumnIdentifier columnIdentifier = dimension.getColumnIdentifier();
    String storePath = carbonTable.getStorePath();
    DictionaryService dictionaryService = CarbonCommonFactory.getDictionaryService();
    // create dictionary cache from dictionary File
    DictionaryColumnUniqueIdentifier identifier = new DictionaryColumnUniqueIdentifier(tableIdentifier, columnIdentifier, columnIdentifier.getDataType());
    Boolean isDictExists = CarbonUtil.isFileExistsForGivenColumn(storePath, identifier);
    Dictionary dictionary = null;
    long t1 = System.currentTimeMillis();
    if (isDictExists) {
        Cache<DictionaryColumnUniqueIdentifier, Dictionary> dictCache = CacheProvider.getInstance().createCache(CacheType.REVERSE_DICTIONARY, storePath);
        dictionary = dictCache.get(identifier);
    }
    long dictCacheTime = System.currentTimeMillis() - t1;
    long t2 = System.currentTimeMillis();
    // write dictionary
    CarbonDictionaryWriter dictionaryWriter = null;
    dictionaryWriter = dictionaryService.getDictionaryWriter(tableIdentifier, columnIdentifier, storePath);
    List<String> distinctValues = writeDictionary(dictionaryWriter, isDictExists);
    long dictWriteTime = System.currentTimeMillis() - t2;
    long t3 = System.currentTimeMillis();
    // write sort index
    if (distinctValues.size() > 0) {
        writeSortIndex(distinctValues, dictionary, dictionaryService, tableIdentifier, columnIdentifier, storePath);
    }
    long sortIndexWriteTime = System.currentTimeMillis() - t3;
    // update Meta Data
    updateMetaData(dictionaryWriter);
    LOGGER.audit("\n columnName: " + dimension.getColName() + "\n columnId: " + dimension.getColumnId() + "\n new distinct values count: " + distinctValues.size() + "\n create dictionary cache: " + dictCacheTime + "\n sort list, distinct and write: " + dictWriteTime + "\n write sort info: " + sortIndexWriteTime);
}

Example 3 with DictionaryColumnUniqueIdentifier

use of org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier in project carbondata by apache.

the class QueryUtil method getDictionaryColumnUniqueIdentifierList.

/**
   * Below method will be used to get the dictionary column unique identifier
   *
   * @param dictionaryColumnIdList dictionary
   * @param carbonTableIdentifier
   * @return
   */
private static List<DictionaryColumnUniqueIdentifier> getDictionaryColumnUniqueIdentifierList(List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier) {
    CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(carbonTableIdentifier.getTableUniqueName());
    List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers = new ArrayList<>(dictionaryColumnIdList.size());
    for (String columnId : dictionaryColumnIdList) {
        CarbonDimension dimension = CarbonMetadata.getInstance().getCarbonDimensionBasedOnColIdentifier(carbonTable, columnId);
        if (dimension != null) {
            dictionaryColumnUniqueIdentifiers.add(new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, dimension.getColumnIdentifier(), dimension.getDataType()));
        }
    }
    return dictionaryColumnUniqueIdentifiers;
}

Example 4 with DictionaryColumnUniqueIdentifier

use of org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier in project carbondata by apache.

the class CacheProvider method createDictionaryCacheForGivenType.

/**
   * This method will create the cache for given cache type
   *
   * @param cacheType       type of cache
   * @param carbonStorePath store path
   */
private void createDictionaryCacheForGivenType(CacheType cacheType, String carbonStorePath) {
    Cache cacheObject = null;
    if (cacheType.equals(CacheType.REVERSE_DICTIONARY)) {
        cacheObject = new ReverseDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath, carbonLRUCache);
    } else if (cacheType.equals(CacheType.FORWARD_DICTIONARY)) {
        cacheObject = new ForwardDictionaryCache<DictionaryColumnUniqueIdentifier, Dictionary>(carbonStorePath, carbonLRUCache);
    } else if (cacheType.equals(cacheType.EXECUTOR_BTREE)) {
        cacheObject = new BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex>(carbonStorePath, carbonLRUCache);
    } else if (cacheType.equals(cacheType.DRIVER_BTREE)) {
        cacheObject = new SegmentTaskIndexStore(carbonStorePath, carbonLRUCache);
    }
    cacheTypeToCacheMap.put(cacheType, cacheObject);
}

Example 5 with DictionaryColumnUniqueIdentifier

use of org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier in project carbondata by apache.

the class QueryUtil method getDictionaryColumnUniqueIdentifierList.

/**
 * Below method will be used to get the dictionary column unique identifier
 *
 * @param dictionaryColumnIdList dictionary
 * @param carbonTableIdentifier
 * @return
 */
private static List<DictionaryColumnUniqueIdentifier> getDictionaryColumnUniqueIdentifierList(List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier, TableProvider tableProvider) throws IOException {
    CarbonTable carbonTable = tableProvider.getCarbonTable(carbonTableIdentifier);
    List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers = new ArrayList<>(dictionaryColumnIdList.size());
    for (String columnId : dictionaryColumnIdList) {
        CarbonDimension dimension = CarbonMetadata.getInstance().getCarbonDimensionBasedOnColIdentifier(carbonTable, columnId);
        if (dimension != null) {
            AbsoluteTableIdentifier dictionarySourceAbsoluteTableIdentifier;
            ColumnIdentifier columnIdentifier;
            if (null != dimension.getColumnSchema().getParentColumnTableRelations() && !dimension.getColumnSchema().getParentColumnTableRelations().isEmpty()) {
                dictionarySourceAbsoluteTableIdentifier = getTableIdentifierForColumn(dimension, carbonTable.getAbsoluteTableIdentifier());
                columnIdentifier = new ColumnIdentifier(dimension.getColumnSchema().getParentColumnTableRelations().get(0).getColumnId(), dimension.getColumnProperties(), dimension.getDataType());
            } else {
                dictionarySourceAbsoluteTableIdentifier = carbonTable.getAbsoluteTableIdentifier();
                columnIdentifier = dimension.getColumnIdentifier();
            }
            String dictionaryPath = carbonTable.getTableInfo().getFactTable().getTableProperties().get(CarbonCommonConstants.DICTIONARY_PATH);
            dictionaryColumnUniqueIdentifiers.add(new DictionaryColumnUniqueIdentifier(dictionarySourceAbsoluteTableIdentifier, columnIdentifier, dimension.getDataType(), dictionaryPath));
        }
    }
    return dictionaryColumnUniqueIdentifiers;
}