use of org.apache.carbondata.core.scan.model.QueryDimension in project carbondata by apache.
the class AbstractQueryExecutor method getBlockExecutionInfoForBlock.
/**
* Below method will be used to get the block execution info which is
* required to execute any block based on query model
*
* @param queryModel query model from user query
* @param blockIndex block index
* @return block execution info
* @throws QueryExecutionException any failure during block info creation
*/
protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel, AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath) throws QueryExecutionException {
BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
List<CarbonDimension> tableBlockDimensions = segmentProperties.getDimensions();
KeyGenerator blockKeyGenerator = segmentProperties.getDimensionKeyGenerator();
// below is to get only those dimension in query which is present in the
// table block
List<QueryDimension> currentBlockQueryDimensions = RestructureUtil.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo, queryModel.getQueryDimension(), tableBlockDimensions, segmentProperties.getComplexDimensions());
int tableFactPathLength = CarbonStorePath.getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getStorePath(), queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier()).getFactDir().length() + 1;
blockExecutionInfo.setBlockId(filePath.substring(tableFactPathLength));
blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
blockExecutionInfo.setQueryDimensions(currentBlockQueryDimensions.toArray(new QueryDimension[currentBlockQueryDimensions.size()]));
// get measures present in the current block
List<QueryMeasure> currentBlockQueryMeasures = getCurrentBlockQueryMeasures(blockExecutionInfo, queryModel, blockIndex);
blockExecutionInfo.setQueryMeasures(currentBlockQueryMeasures.toArray(new QueryMeasure[currentBlockQueryMeasures.size()]));
blockExecutionInfo.setDataBlock(blockIndex);
blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
// setting whether raw record query or not
blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
// total number dimension
blockExecutionInfo.setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
blockExecutionInfo.setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil.getComplexDimensionsMap(currentBlockQueryDimensions, segmentProperties.getDimensionOrdinalToBlockMapping(), segmentProperties.getEachComplexDimColumnValueSize(), queryProperties.columnToDictionayMapping, queryProperties.complexFilterDimension));
IndexKey startIndexKey = null;
IndexKey endIndexKey = null;
if (null != queryModel.getFilterExpressionResolverTree()) {
// loading the filter executer tree for filter evaluation
blockExecutionInfo.setFilterExecuterTree(FilterUtil.getFilterExecuterTree(queryModel.getFilterExpressionResolverTree(), segmentProperties, blockExecutionInfo.getComlexDimensionInfoMap()));
List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
FilterUtil.traverseResolverTreeAndGetStartAndEndKey(segmentProperties, queryModel.getFilterExpressionResolverTree(), listOfStartEndKeys);
startIndexKey = listOfStartEndKeys.get(0);
endIndexKey = listOfStartEndKeys.get(1);
} else {
try {
startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
} catch (KeyGenException e) {
throw new QueryExecutionException(e);
}
}
//setting the start index key of the block node
blockExecutionInfo.setStartKey(startIndexKey);
//setting the end index key of the block node
blockExecutionInfo.setEndKey(endIndexKey);
// expression dimensions
List<CarbonDimension> expressionDimensions = new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
// expression measure
List<CarbonMeasure> expressionMeasures = new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
// setting all the dimension chunk indexes to be read from file
int numberOfElementToConsider = 0;
// list of dimensions to be projected
Set<Integer> allProjectionListDimensionIdexes = new LinkedHashSet<>();
// create a list of filter dimensions present in the current block
Set<CarbonDimension> currentBlockFilterDimensions = getCurrentBlockFilterDimensions(queryProperties.complexFilterDimension, segmentProperties);
int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(currentBlockQueryDimensions, segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions, currentBlockFilterDimensions, allProjectionListDimensionIdexes);
int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance().getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO, CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE));
if (dimensionsBlockIndexes.length > 0) {
numberOfElementToConsider = dimensionsBlockIndexes[dimensionsBlockIndexes.length - 1] == segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ? dimensionsBlockIndexes.length - 1 : dimensionsBlockIndexes.length;
blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(CarbonUtil.getRangeIndex(dimensionsBlockIndexes, numberOfElementToConsider, numberOfColumnToBeReadInOneIO));
} else {
blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[0][0]);
}
// get the list of updated filter measures present in the current block
Set<CarbonMeasure> currentBlockFilterMeasures = getCurrentBlockFilterMeasures(queryProperties.filterMeasures, segmentProperties);
// list of measures to be projected
List<Integer> allProjectionListMeasureIndexes = new ArrayList<>();
int[] measureBlockIndexes = QueryUtil.getMeasureBlockIndexes(currentBlockQueryMeasures, expressionMeasures, segmentProperties.getMeasuresOrdinalToBlockMapping(), currentBlockFilterMeasures, allProjectionListMeasureIndexes);
if (measureBlockIndexes.length > 0) {
numberOfElementToConsider = measureBlockIndexes[measureBlockIndexes.length - 1] == segmentProperties.getMeasures().size() - 1 ? measureBlockIndexes.length - 1 : measureBlockIndexes.length;
// setting all the measure chunk indexes to be read from file
blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(CarbonUtil.getRangeIndex(measureBlockIndexes, numberOfElementToConsider, numberOfColumnToBeReadInOneIO));
} else {
blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[0][0]);
}
// setting the indexes of list of dimension in projection list
blockExecutionInfo.setProjectionListDimensionIndexes(ArrayUtils.toPrimitive(allProjectionListDimensionIdexes.toArray(new Integer[allProjectionListDimensionIdexes.size()])));
// setting the indexes of list of measures in projection list
blockExecutionInfo.setProjectionListMeasureIndexes(ArrayUtils.toPrimitive(allProjectionListMeasureIndexes.toArray(new Integer[allProjectionListMeasureIndexes.size()])));
// setting the size of fixed key column (dictionary column)
blockExecutionInfo.setFixedLengthKeySize(getKeySize(currentBlockQueryDimensions, segmentProperties));
Set<Integer> dictionaryColumnBlockIndex = new HashSet<Integer>();
List<Integer> noDictionaryColumnBlockIndex = new ArrayList<Integer>();
// get the block index to be read from file for query dimension
// for both dictionary columns and no dictionary columns
QueryUtil.fillQueryDimensionsBlockIndexes(currentBlockQueryDimensions, segmentProperties.getDimensionOrdinalToBlockMapping(), dictionaryColumnBlockIndex, noDictionaryColumnBlockIndex);
int[] queryDictionaryColumnBlockIndexes = ArrayUtils.toPrimitive(dictionaryColumnBlockIndex.toArray(new Integer[dictionaryColumnBlockIndex.size()]));
// need to sort the dictionary column as for all dimension
// column key will be filled based on key order
Arrays.sort(queryDictionaryColumnBlockIndexes);
blockExecutionInfo.setDictionaryColumnBlockIndex(queryDictionaryColumnBlockIndexes);
// setting the no dictionary column block indexes
blockExecutionInfo.setNoDictionaryBlockIndexes(ArrayUtils.toPrimitive(noDictionaryColumnBlockIndex.toArray(new Integer[noDictionaryColumnBlockIndex.size()])));
// setting column id to dictionary mapping
blockExecutionInfo.setColumnIdToDcitionaryMapping(queryProperties.columnToDictionayMapping);
// setting each column value size
blockExecutionInfo.setEachColumnValueSize(segmentProperties.getEachDimColumnValueSize());
blockExecutionInfo.setComplexColumnParentBlockIndexes(getComplexDimensionParentBlockIndexes(currentBlockQueryDimensions));
blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
try {
// to set column group and its key structure info which will be used
// to
// for getting the column group column data in case of final row
// and in case of dimension aggregation
blockExecutionInfo.setColumnGroupToKeyStructureInfo(QueryUtil.getColumnGroupKeyStructureInfo(currentBlockQueryDimensions, segmentProperties));
} catch (KeyGenException e) {
throw new QueryExecutionException(e);
}
// set actual query dimensions and measures. It may differ in case of restructure scenarios
blockExecutionInfo.setActualQueryDimensions(queryModel.getQueryDimension().toArray(new QueryDimension[queryModel.getQueryDimension().size()]));
blockExecutionInfo.setActualQueryMeasures(queryModel.getQueryMeasures().toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
return blockExecutionInfo;
}
use of org.apache.carbondata.core.scan.model.QueryDimension in project carbondata by apache.
the class QueryUtil method resolveQueryModel.
/**
* Below method will be used to resolve the query model
* resolve will be setting the actual dimension and measure object
* as from driver only column name will be passes to avoid the heavy object
* serialization
*
* @param queryModel query model
*/
public static void resolveQueryModel(QueryModel queryModel) {
CarbonMetadata.getInstance().addCarbonTable(queryModel.getTable());
// TODO need to load the table from table identifier
CarbonTable carbonTable = queryModel.getTable();
String tableName = queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName();
// resolve query dimension
for (QueryDimension queryDimension : queryModel.getQueryDimension()) {
queryDimension.setDimension(carbonTable.getDimensionByName(tableName, queryDimension.getColumnName()));
}
// resolve query measure
for (QueryMeasure queryMeasure : queryModel.getQueryMeasures()) {
//never come false but if in future we can remove so not removing first if check
if (queryMeasure.getColumnName().equals("count(*)")) {
if (carbonTable.getMeasureByTableName(tableName).size() > 0 && !carbonTable.getMeasureByTableName(tableName).get(0).getColName().equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)) {
queryMeasure.setMeasure(carbonTable.getMeasureByTableName(tableName).get(0));
} else {
CarbonMeasure dummyMeasure = new CarbonMeasure(carbonTable.getDimensionByTableName(tableName).get(0).getColumnSchema(), 0);
queryMeasure.setMeasure(dummyMeasure);
}
} else {
queryMeasure.setMeasure(carbonTable.getMeasureByName(tableName, queryMeasure.getColumnName()));
}
}
}
use of org.apache.carbondata.core.scan.model.QueryDimension in project carbondata by apache.
the class CarbonCompactionExecutor method prepareQueryModel.
/**
* Preparing of the query model.
*
* @param blockList
* @return
*/
private QueryModel prepareQueryModel(List<TableBlockInfo> blockList) {
QueryModel model = new QueryModel();
model.setTableBlockInfos(blockList);
model.setForcedDetailRawQuery(true);
model.setFilterExpressionResolverTree(null);
List<QueryDimension> dims = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(carbonTable.getFactTableName());
for (CarbonDimension dim : dimensions) {
// check if dimension is deleted
QueryDimension queryDimension = new QueryDimension(dim.getColName());
queryDimension.setDimension(dim);
dims.add(queryDimension);
}
model.setQueryDimension(dims);
List<QueryMeasure> msrs = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(carbonTable.getFactTableName());
for (CarbonMeasure carbonMeasure : measures) {
// check if measure is deleted
QueryMeasure queryMeasure = new QueryMeasure(carbonMeasure.getColName());
queryMeasure.setMeasure(carbonMeasure);
msrs.add(queryMeasure);
}
model.setQueryMeasures(msrs);
model.setQueryId(System.nanoTime() + "");
model.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
model.setTable(carbonTable);
return model;
}
use of org.apache.carbondata.core.scan.model.QueryDimension in project carbondata by apache.
the class CarbonInputFormatUtil method addQueryDimension.
private static void addQueryDimension(CarbonQueryPlan plan, int order, CarbonDimension dimension) {
QueryDimension queryDimension = new QueryDimension(dimension.getColName());
queryDimension.setQueryOrder(order);
queryDimension.setDimension(dimension);
plan.addDimension(queryDimension);
}
use of org.apache.carbondata.core.scan.model.QueryDimension in project carbondata by apache.
the class RestructureUtil method createDimensionInfoAndGetCurrentBlockQueryDimension.
/**
* Below method will be used to get the updated query dimension updation
* means, after restructuring some dimension will be not present in older
* table blocks in that case we need to select only those dimension out of
* query dimension which is present in the current table block
*
* @param blockExecutionInfo
* @param queryDimensions
* @param tableBlockDimensions
* @param tableComplexDimension
* @return list of query dimension which is present in the table block
*/
public static List<QueryDimension> createDimensionInfoAndGetCurrentBlockQueryDimension(BlockExecutionInfo blockExecutionInfo, List<QueryDimension> queryDimensions, List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension) {
List<QueryDimension> presentDimension = new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
boolean[] isDimensionExists = new boolean[queryDimensions.size()];
Object[] defaultValues = new Object[queryDimensions.size()];
// create dimension information instance
DimensionInfo dimensionInfo = new DimensionInfo(isDimensionExists, defaultValues);
int newDictionaryColumnCount = 0;
int newNoDictionaryColumnCount = 0;
// selecting only those dimension which is present in the query
int dimIndex = 0;
for (QueryDimension queryDimension : queryDimensions) {
if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
presentDimension.add(queryDimension);
isDimensionExists[dimIndex] = true;
} else {
for (CarbonDimension tableDimension : tableBlockDimensions) {
if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
QueryDimension currentBlockDimension = new QueryDimension(tableDimension.getColName());
tableDimension.getColumnSchema().setDataType(queryDimension.getDimension().getDataType());
tableDimension.getColumnSchema().setPrecision(queryDimension.getDimension().getColumnSchema().getPrecision());
tableDimension.getColumnSchema().setScale(queryDimension.getDimension().getColumnSchema().getScale());
tableDimension.getColumnSchema().setDefaultValue(queryDimension.getDimension().getDefaultValue());
currentBlockDimension.setDimension(tableDimension);
currentBlockDimension.setQueryOrder(queryDimension.getQueryOrder());
presentDimension.add(currentBlockDimension);
isDimensionExists[dimIndex] = true;
break;
}
}
// if dimension is found then no need to search in the complex dimensions list
if (isDimensionExists[dimIndex]) {
dimIndex++;
continue;
}
for (CarbonDimension tableDimension : tableComplexDimension) {
if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
QueryDimension currentBlockDimension = new QueryDimension(tableDimension.getColName());
// TODO: for complex dimension set scale and precision by traversing
// the child dimensions
currentBlockDimension.setDimension(tableDimension);
currentBlockDimension.setQueryOrder(queryDimension.getQueryOrder());
presentDimension.add(currentBlockDimension);
isDimensionExists[dimIndex] = true;
break;
}
}
// add default value only in case query dimension is not found in the current block
if (!isDimensionExists[dimIndex]) {
defaultValues[dimIndex] = validateAndGetDefaultValue(queryDimension.getDimension());
blockExecutionInfo.setRestructuredBlock(true);
// newly added columns data need to be filled
if (queryDimension.getDimension().hasEncoding(Encoding.DICTIONARY)) {
dimensionInfo.setDictionaryColumnAdded(true);
newDictionaryColumnCount++;
} else {
dimensionInfo.setNoDictionaryColumnAdded(true);
newNoDictionaryColumnCount++;
}
}
}
dimIndex++;
}
dimensionInfo.setNewDictionaryColumnCount(newDictionaryColumnCount);
dimensionInfo.setNewNoDictionaryColumnCount(newNoDictionaryColumnCount);
blockExecutionInfo.setDimensionInfo(dimensionInfo);
return presentDimension;
}
Aggregations