Examples with CacheProvider org.apache.carbondata.core.cache.CacheProvider used on opensource projects

Example 1 with CacheProvider

use of org.apache.carbondata.core.cache.CacheProvider in project carbondata by apache.

the class AbstractQueryExecutor method initQuery.

/**
   * Below method will be used to fill the executor properties based on query
   * model it will parse the query model and get the detail and fill it in
   * query properties
   *
   * @param queryModel
   */
protected void initQuery(QueryModel queryModel) throws IOException {
    StandardLogService.setThreadName(StandardLogService.getPartitionID(queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName()), queryModel.getQueryId());
    LOGGER.info("Query will be executed on table: " + queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName());
    // add executor service for query execution
    queryProperties.executorService = Executors.newCachedThreadPool();
    // Initializing statistics list to record the query statistics
    // creating copy on write to handle concurrent scenario
    queryProperties.queryStatisticsRecorder = CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId());
    queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
    QueryUtil.resolveQueryModel(queryModel);
    QueryStatistic queryStatistic = new QueryStatistic();
    // sort the block info
    // so block will be loaded in sorted order this will be required for
    // query execution
    Collections.sort(queryModel.getTableBlockInfos());
    // get the table blocks
    CacheProvider cacheProvider = CacheProvider.getInstance();
    BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex> cache = (BlockIndexStore) cacheProvider.createCache(CacheType.EXECUTOR_BTREE, queryModel.getTable().getStorePath());
    // remove the invalid table blocks, block which is deleted or compacted
    cache.removeTableBlocks(queryModel.getInvalidSegmentIds(), queryModel.getAbsoluteTableIdentifier());
    List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers = prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(), queryModel.getAbsoluteTableIdentifier());
    cache.removeTableBlocksIfHorizontalCompactionDone(queryModel);
    queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
    queryStatistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
    queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
    // calculating the total number of aggeragted columns
    int aggTypeCount = queryModel.getQueryMeasures().size();
    int currentIndex = 0;
    DataType[] dataTypes = new DataType[aggTypeCount];
    for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
        // adding the data type and aggregation type of all the measure this
        // can be used
        // to select the aggregator
        dataTypes[currentIndex] = carbonMeasure.getMeasure().getDataType();
        currentIndex++;
    }
    queryProperties.measureDataTypes = dataTypes;
    // as aggregation will be executed in following order
    // 1.aggregate dimension expression
    // 2. expression
    // 3. query measure
    // so calculating the index of the expression start index
    // and measure column start index
    queryProperties.filterMeasures = new HashSet<>();
    queryProperties.complexFilterDimension = new HashSet<>();
    QueryUtil.getAllFilterDimensions(queryModel.getFilterExpressionResolverTree(), queryProperties.complexFilterDimension, queryProperties.filterMeasures);
    queryStatistic = new QueryStatistic();
    // dictionary column unique column id to dictionary mapping
    // which will be used to get column actual data
    queryProperties.columnToDictionayMapping = QueryUtil.getDimensionDictionaryDetail(queryModel.getQueryDimension(), queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier());
    queryStatistic.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis());
    queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
    queryModel.setColumnToDictionaryMapping(queryProperties.columnToDictionayMapping);
}

Example 2 with CacheProvider

use of org.apache.carbondata.core.cache.CacheProvider in project carbondata by apache.

the class ReverseDictionaryCacheTest method createDictionaryCacheObject.

private void createDictionaryCacheObject() {
    // enable lru cache by setting cache size
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10");
    CacheProvider cacheProvider = CacheProvider.getInstance();
    cacheProvider.dropAllCache();
    reverseDictionaryCache = cacheProvider.createCache(CacheType.REVERSE_DICTIONARY, this.carbonStorePath);
}

Example 3 with CacheProvider

use of org.apache.carbondata.core.cache.CacheProvider in project carbondata by apache.

the class FilterUtil method getForwardDictionaryCache.

/**
   * @param tableIdentifier
   * @param carbonDimension
   * @return
   */
public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier, CarbonDimension carbonDimension) throws IOException {
    DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier = new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(), carbonDimension.getColumnIdentifier(), carbonDimension.getDataType());
    CacheProvider cacheProvider = CacheProvider.getInstance();
    Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, tableIdentifier.getStorePath());
    // get the forward dictionary object
    return forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
}

Example 4 with CacheProvider

use of org.apache.carbondata.core.cache.CacheProvider in project carbondata by apache.

the class BlockIndexStoreTest method setUp.

@BeforeClass
public void setUp() {
    property = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION);
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_DATA_FILE_VERSION, "1");
    StoreCreator.createCarbonStore();
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10");
    CacheProvider cacheProvider = CacheProvider.getInstance();
    cache = (BlockIndexStore) cacheProvider.createCache(CacheType.EXECUTOR_BTREE, "");
}

Example 5 with CacheProvider

use of org.apache.carbondata.core.cache.CacheProvider in project carbondata by apache.

the class QueryUtil method getDictionaryMap.

/**
   * Below method will be used to get the column id to its dictionary mapping
   *
   * @param dictionaryColumnIdList  dictionary column list
   * @param absoluteTableIdentifier absolute table identifier
   * @return dictionary mapping
   * @throws IOException
   */
private static Map<String, Dictionary> getDictionaryMap(List<String> dictionaryColumnIdList, AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
    // this for dictionary unique identifier
    List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers = getDictionaryColumnUniqueIdentifierList(dictionaryColumnIdList, absoluteTableIdentifier.getCarbonTableIdentifier());
    CacheProvider cacheProvider = CacheProvider.getInstance();
    Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath());
    List<Dictionary> columnDictionaryList = forwardDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
    Map<String, Dictionary> columnDictionaryMap = new HashMap<>(columnDictionaryList.size());
    for (int i = 0; i < dictionaryColumnUniqueIdentifiers.size(); i++) {
        // TODO: null check for column dictionary, if cache size is less it
        // might return null here, in that case throw exception
        columnDictionaryMap.put(dictionaryColumnIdList.get(i), columnDictionaryList.get(i));
    }
    return columnDictionaryMap;
}