use of org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo in project carbondata by apache.
the class AbstractQueryExecutor method getBlockExecutionInfoForBlock.
/**
* Below method will be used to get the block execution info which is
* required to execute any block based on query model
*
* @param queryModel query model from user query
* @param blockIndex block index
* @return block execution info
* @throws QueryExecutionException any failure during block info creation
*/
protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel, AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath) throws QueryExecutionException {
BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
List<CarbonDimension> tableBlockDimensions = segmentProperties.getDimensions();
KeyGenerator blockKeyGenerator = segmentProperties.getDimensionKeyGenerator();
// below is to get only those dimension in query which is present in the
// table block
List<QueryDimension> currentBlockQueryDimensions = RestructureUtil.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo, queryModel.getQueryDimension(), tableBlockDimensions, segmentProperties.getComplexDimensions());
int tableFactPathLength = CarbonStorePath.getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getStorePath(), queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier()).getFactDir().length() + 1;
blockExecutionInfo.setBlockId(filePath.substring(tableFactPathLength));
blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
blockExecutionInfo.setQueryDimensions(currentBlockQueryDimensions.toArray(new QueryDimension[currentBlockQueryDimensions.size()]));
// get measures present in the current block
List<QueryMeasure> currentBlockQueryMeasures = getCurrentBlockQueryMeasures(blockExecutionInfo, queryModel, blockIndex);
blockExecutionInfo.setQueryMeasures(currentBlockQueryMeasures.toArray(new QueryMeasure[currentBlockQueryMeasures.size()]));
blockExecutionInfo.setDataBlock(blockIndex);
blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
// setting whether raw record query or not
blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
// total number dimension
blockExecutionInfo.setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
blockExecutionInfo.setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil.getComplexDimensionsMap(currentBlockQueryDimensions, segmentProperties.getDimensionOrdinalToBlockMapping(), segmentProperties.getEachComplexDimColumnValueSize(), queryProperties.columnToDictionayMapping, queryProperties.complexFilterDimension));
IndexKey startIndexKey = null;
IndexKey endIndexKey = null;
if (null != queryModel.getFilterExpressionResolverTree()) {
// loading the filter executer tree for filter evaluation
blockExecutionInfo.setFilterExecuterTree(FilterUtil.getFilterExecuterTree(queryModel.getFilterExpressionResolverTree(), segmentProperties, blockExecutionInfo.getComlexDimensionInfoMap()));
List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
FilterUtil.traverseResolverTreeAndGetStartAndEndKey(segmentProperties, queryModel.getFilterExpressionResolverTree(), listOfStartEndKeys);
startIndexKey = listOfStartEndKeys.get(0);
endIndexKey = listOfStartEndKeys.get(1);
} else {
try {
startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
} catch (KeyGenException e) {
throw new QueryExecutionException(e);
}
}
//setting the start index key of the block node
blockExecutionInfo.setStartKey(startIndexKey);
//setting the end index key of the block node
blockExecutionInfo.setEndKey(endIndexKey);
// expression dimensions
List<CarbonDimension> expressionDimensions = new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
// expression measure
List<CarbonMeasure> expressionMeasures = new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
// setting all the dimension chunk indexes to be read from file
int numberOfElementToConsider = 0;
// list of dimensions to be projected
Set<Integer> allProjectionListDimensionIdexes = new LinkedHashSet<>();
// create a list of filter dimensions present in the current block
Set<CarbonDimension> currentBlockFilterDimensions = getCurrentBlockFilterDimensions(queryProperties.complexFilterDimension, segmentProperties);
int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(currentBlockQueryDimensions, segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions, currentBlockFilterDimensions, allProjectionListDimensionIdexes);
int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance().getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO, CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE));
if (dimensionsBlockIndexes.length > 0) {
numberOfElementToConsider = dimensionsBlockIndexes[dimensionsBlockIndexes.length - 1] == segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ? dimensionsBlockIndexes.length - 1 : dimensionsBlockIndexes.length;
blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(CarbonUtil.getRangeIndex(dimensionsBlockIndexes, numberOfElementToConsider, numberOfColumnToBeReadInOneIO));
} else {
blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[0][0]);
}
// get the list of updated filter measures present in the current block
Set<CarbonMeasure> currentBlockFilterMeasures = getCurrentBlockFilterMeasures(queryProperties.filterMeasures, segmentProperties);
// list of measures to be projected
List<Integer> allProjectionListMeasureIndexes = new ArrayList<>();
int[] measureBlockIndexes = QueryUtil.getMeasureBlockIndexes(currentBlockQueryMeasures, expressionMeasures, segmentProperties.getMeasuresOrdinalToBlockMapping(), currentBlockFilterMeasures, allProjectionListMeasureIndexes);
if (measureBlockIndexes.length > 0) {
numberOfElementToConsider = measureBlockIndexes[measureBlockIndexes.length - 1] == segmentProperties.getMeasures().size() - 1 ? measureBlockIndexes.length - 1 : measureBlockIndexes.length;
// setting all the measure chunk indexes to be read from file
blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(CarbonUtil.getRangeIndex(measureBlockIndexes, numberOfElementToConsider, numberOfColumnToBeReadInOneIO));
} else {
blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[0][0]);
}
// setting the indexes of list of dimension in projection list
blockExecutionInfo.setProjectionListDimensionIndexes(ArrayUtils.toPrimitive(allProjectionListDimensionIdexes.toArray(new Integer[allProjectionListDimensionIdexes.size()])));
// setting the indexes of list of measures in projection list
blockExecutionInfo.setProjectionListMeasureIndexes(ArrayUtils.toPrimitive(allProjectionListMeasureIndexes.toArray(new Integer[allProjectionListMeasureIndexes.size()])));
// setting the size of fixed key column (dictionary column)
blockExecutionInfo.setFixedLengthKeySize(getKeySize(currentBlockQueryDimensions, segmentProperties));
Set<Integer> dictionaryColumnBlockIndex = new HashSet<Integer>();
List<Integer> noDictionaryColumnBlockIndex = new ArrayList<Integer>();
// get the block index to be read from file for query dimension
// for both dictionary columns and no dictionary columns
QueryUtil.fillQueryDimensionsBlockIndexes(currentBlockQueryDimensions, segmentProperties.getDimensionOrdinalToBlockMapping(), dictionaryColumnBlockIndex, noDictionaryColumnBlockIndex);
int[] queryDictionaryColumnBlockIndexes = ArrayUtils.toPrimitive(dictionaryColumnBlockIndex.toArray(new Integer[dictionaryColumnBlockIndex.size()]));
// need to sort the dictionary column as for all dimension
// column key will be filled based on key order
Arrays.sort(queryDictionaryColumnBlockIndexes);
blockExecutionInfo.setDictionaryColumnBlockIndex(queryDictionaryColumnBlockIndexes);
// setting the no dictionary column block indexes
blockExecutionInfo.setNoDictionaryBlockIndexes(ArrayUtils.toPrimitive(noDictionaryColumnBlockIndex.toArray(new Integer[noDictionaryColumnBlockIndex.size()])));
// setting column id to dictionary mapping
blockExecutionInfo.setColumnIdToDcitionaryMapping(queryProperties.columnToDictionayMapping);
// setting each column value size
blockExecutionInfo.setEachColumnValueSize(segmentProperties.getEachDimColumnValueSize());
blockExecutionInfo.setComplexColumnParentBlockIndexes(getComplexDimensionParentBlockIndexes(currentBlockQueryDimensions));
blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
try {
// to set column group and its key structure info which will be used
// to
// for getting the column group column data in case of final row
// and in case of dimension aggregation
blockExecutionInfo.setColumnGroupToKeyStructureInfo(QueryUtil.getColumnGroupKeyStructureInfo(currentBlockQueryDimensions, segmentProperties));
} catch (KeyGenException e) {
throw new QueryExecutionException(e);
}
// set actual query dimensions and measures. It may differ in case of restructure scenarios
blockExecutionInfo.setActualQueryDimensions(queryModel.getQueryDimension().toArray(new QueryDimension[queryModel.getQueryDimension().size()]));
blockExecutionInfo.setActualQueryMeasures(queryModel.getQueryMeasures().toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
return blockExecutionInfo;
}
use of org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo in project carbondata by apache.
the class VectorDetailQueryExecutor method execute.
@Override
public CarbonIterator<Object> execute(QueryModel queryModel) throws QueryExecutionException, IOException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
this.queryIterator = new VectorDetailQueryResultIterator(blockExecutionInfoList, queryModel, queryProperties.executorService);
return this.queryIterator;
}
use of org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo in project carbondata by apache.
the class AbstractQueryExecutor method getBlockExecutionInfos.
protected List<BlockExecutionInfo> getBlockExecutionInfos(QueryModel queryModel) throws IOException, QueryExecutionException {
initQuery(queryModel);
List<BlockExecutionInfo> blockExecutionInfoList = new ArrayList<BlockExecutionInfo>();
// and query will be executed based on that infos
for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
AbstractIndex abstractIndex = queryProperties.dataBlocks.get(i);
BlockletDataRefNode dataRefNode = (BlockletDataRefNode) abstractIndex.getDataRefNode();
blockExecutionInfoList.add(getBlockExecutionInfoForBlock(queryModel, abstractIndex, dataRefNode.getBlockInfos().get(0).getBlockletInfos().getStartBlockletNumber(), dataRefNode.numberOfNodes(), dataRefNode.getBlockInfos().get(0).getFilePath(), dataRefNode.getBlockInfos().get(0).getDeletedDeltaFilePath(), dataRefNode.getBlockInfos().get(0).getSegmentId()));
}
if (null != queryModel.getStatisticsRecorder()) {
QueryStatistic queryStatistic = new QueryStatistic();
queryStatistic.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_NUM, blockExecutionInfoList.size());
queryModel.getStatisticsRecorder().recordStatistics(queryStatistic);
}
return blockExecutionInfoList;
}
use of org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo in project carbondata by apache.
the class DetailQueryExecutor method execute.
@Override
public CarbonIterator<RowBatch> execute(QueryModel queryModel) throws QueryExecutionException, IOException {
List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
this.queryIterator = new DetailQueryResultIterator(blockExecutionInfoList, queryModel, queryProperties.executorService);
return queryIterator;
}
use of org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo in project carbondata by apache.
the class RestructureUtilTest method testToGetAggregatorInfos.
@Test
public void testToGetAggregatorInfos() {
ColumnSchema columnSchema1 = new ColumnSchema();
columnSchema1.setColumnName("Id");
columnSchema1.setDataType(DataTypes.STRING);
columnSchema1.setColumnUniqueId(UUID.randomUUID().toString());
ColumnSchema columnSchema2 = new ColumnSchema();
columnSchema2.setColumnName("Name");
columnSchema2.setDataType(DataTypes.STRING);
columnSchema2.setColumnUniqueId(UUID.randomUUID().toString());
ColumnSchema columnSchema3 = new ColumnSchema();
columnSchema3.setColumnName("Age");
columnSchema3.setDataType(DataTypes.STRING);
columnSchema3.setColumnUniqueId(UUID.randomUUID().toString());
CarbonMeasure carbonMeasure1 = new CarbonMeasure(columnSchema1, 1);
CarbonMeasure carbonMeasure2 = new CarbonMeasure(columnSchema2, 2);
CarbonMeasure carbonMeasure3 = new CarbonMeasure(columnSchema3, 3);
carbonMeasure3.getColumnSchema().setDefaultValue("3".getBytes());
List<CarbonMeasure> currentBlockMeasures = Arrays.asList(carbonMeasure1, carbonMeasure2);
ProjectionMeasure queryMeasure1 = new ProjectionMeasure(carbonMeasure1);
ProjectionMeasure queryMeasure2 = new ProjectionMeasure(carbonMeasure2);
ProjectionMeasure queryMeasure3 = new ProjectionMeasure(carbonMeasure3);
List<ProjectionMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2, queryMeasure3);
BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
RestructureUtil.createMeasureInfoAndGetCurrentBlockQueryMeasures(blockExecutionInfo, queryMeasures, currentBlockMeasures);
MeasureInfo measureInfo = blockExecutionInfo.getMeasureInfo();
boolean[] measuresExist = { true, true, false };
assertThat(measureInfo.getMeasureExists(), is(equalTo(measuresExist)));
Object[] defaultValues = { null, null, 3.0 };
assertThat(measureInfo.getDefaultValues(), is(equalTo(defaultValues)));
}
Aggregations