use of org.apache.drill.exec.planner.index.IndexProperties in project drill by apache.
the class DbScanSortRemovalRule method doOnMatch.
private void doOnMatch(IndexPhysicalPlanCallContext indexContext) {
Stopwatch indexPlanTimer = Stopwatch.createStarted();
final PlannerSettings settings = PrelUtil.getPlannerSettings(indexContext.call.getPlanner());
DbGroupScan groupScan = (DbGroupScan) indexContext.scan.getGroupScan();
boolean isIndexScan = groupScan.isIndexScan();
if (!isIndexScan) {
// This case generates the index scan and removes the sort if possible.
final IndexCollection indexCollection = groupScan.getSecondaryIndexCollection(indexContext.scan);
if (indexCollection == null) {
return;
}
if (settings.isStatisticsEnabled()) {
groupScan.getStatistics().initialize(null, indexContext.scan, indexContext);
}
IndexPlanUtils.updateSortExpression(indexContext, indexContext.getSort() != null ? indexContext.getCollation().getFieldCollations() : null);
IndexSelector selector = new IndexSelector(indexContext);
for (IndexDescriptor indexDesc : indexCollection) {
indexDesc.getIndexGroupScan().setStatistics(groupScan.getStatistics());
FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo();
if (IndexPlanUtils.isCoveringIndex(indexContext, functionInfo)) {
selector.addIndex(indexDesc, true, indexContext.lowerProject != null ? indexContext.lowerProject.getRowType().getFieldCount() : indexContext.scan.getRowType().getFieldCount());
}
}
IndexProperties idxProp = selector.getBestIndexNoFilter();
if (idxProp != null) {
try {
// generate a covering plan
CoveringPlanNoFilterGenerator planGen = new CoveringPlanNoFilterGenerator(indexContext, idxProp.getIndexDesc().getFunctionalInfo(), false, settings);
if (planGen.convertChild() != null) {
indexContext.getCall().transformTo(planGen.convertChild());
} else {
logger.debug("Not able to generate index plan in {}", this.getClass().toString());
}
} catch (Exception e) {
logger.warn("Exception while trying to generate indexscan to remove sort", e);
}
}
} else {
Preconditions.checkNotNull(indexContext.getSort());
// This case tries to use the already generated index to see if a sort can be removed.
if (indexContext.scan.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE).getFieldCollations().size() == 0) {
return;
}
try {
RelNode finalRel = indexContext.scan.copy(indexContext.scan.getTraitSet(), indexContext.scan.getInputs());
if (indexContext.lowerProject != null) {
List<RelNode> inputs = Lists.newArrayList();
inputs.add(finalRel);
finalRel = indexContext.lowerProject.copy(indexContext.lowerProject.getTraitSet(), inputs);
}
finalRel = AbstractIndexPlanGenerator.getSortNode(indexContext, finalRel, true, false, indexContext.exch != null);
if (finalRel == null) {
logger.debug("Not able to generate index plan in {}", this.getClass().toString());
return;
}
finalRel = Prule.convert(finalRel, finalRel.getTraitSet().plus(Prel.DRILL_PHYSICAL));
indexContext.getCall().transformTo(finalRel);
} catch (Exception e) {
logger.warn("Exception while trying to use the indexscan to remove the sort", e);
}
}
indexPlanTimer.stop();
logger.debug("Index Planning took {} ms", indexPlanTimer.elapsed(TimeUnit.MILLISECONDS));
}
use of org.apache.drill.exec.planner.index.IndexProperties in project drill by apache.
the class DbScanToIndexScanPrule method processWithIndexSelection.
private void processWithIndexSelection(IndexLogicalPlanCallContext indexContext, PlannerSettings settings, RexNode condition, IndexCollection collection, RexBuilder builder) {
double totalRows = 0;
double filterRows = totalRows;
DrillScanRel scan = indexContext.scan;
if (!(indexContext.scan.getGroupScan() instanceof DbGroupScan)) {
return;
}
IndexConditionInfo.Builder infoBuilder = IndexConditionInfo.newBuilder(condition, collection, builder, indexContext.scan);
IndexConditionInfo cInfo = infoBuilder.getCollectiveInfo(indexContext);
boolean isValidIndexHint = infoBuilder.isValidIndexHint(indexContext);
if (!cInfo.hasIndexCol) {
logger.info("index_plan_info: No index columns are projected from the scan..continue.");
return;
}
if (cInfo.indexCondition == null) {
logger.info("index_plan_info: No conditions were found eligible for applying index lookup.");
return;
}
if (!indexContext.indexHint.equals("") && !isValidIndexHint) {
logger.warn("index_plan_info: Index Hint {} is not useful as index with that name is not available", indexContext.indexHint);
}
RexNode indexCondition = cInfo.indexCondition;
RexNode remainderCondition = cInfo.remainderCondition;
if (remainderCondition.isAlwaysTrue()) {
remainderCondition = null;
}
logger.debug("index_plan_info: condition split into indexcondition: {} and remaindercondition: {}", indexCondition, remainderCondition);
IndexableExprMarker indexableExprMarker = new IndexableExprMarker(indexContext.scan);
indexCondition.accept(indexableExprMarker);
indexContext.origMarker = indexableExprMarker;
if (scan.getGroupScan() instanceof DbGroupScan) {
// Initialize statistics
DbGroupScan dbScan = ((DbGroupScan) scan.getGroupScan());
if (settings.isStatisticsEnabled()) {
dbScan.getStatistics().initialize(condition, scan, indexContext);
}
totalRows = dbScan.getRowCount(null, scan);
filterRows = dbScan.getRowCount(condition, scan);
double sel = filterRows / totalRows;
if (totalRows != Statistics.ROWCOUNT_UNKNOWN && filterRows != Statistics.ROWCOUNT_UNKNOWN && !settings.isDisableFullTableScan() && !isValidIndexHint && sel > Math.max(settings.getIndexCoveringSelThreshold(), settings.getIndexNonCoveringSelThreshold())) {
// If full table scan is not disabled, generate full table scan only plans if selectivity
// is greater than covering and non-covering selectivity thresholds
logger.info("index_plan_info: Skip index planning because filter selectivity: {} is greater than thresholds {}, {}", sel, settings.getIndexCoveringSelThreshold(), settings.getIndexNonCoveringSelThreshold());
return;
}
}
if (totalRows == Statistics.ROWCOUNT_UNKNOWN || totalRows == 0 || filterRows == Statistics.ROWCOUNT_UNKNOWN) {
logger.warn("index_plan_info: Total row count is UNKNOWN or 0, or filterRows UNKNOWN; skip index planning");
return;
}
List<IndexGroup> coveringIndexes = Lists.newArrayList();
List<IndexGroup> nonCoveringIndexes = Lists.newArrayList();
List<IndexGroup> intersectIndexes = Lists.newArrayList();
// update sort expressions in context, it is needed for computing collation, so do it before IndexSelector
IndexPlanUtils.updateSortExpression(indexContext, indexContext.sort != null ? indexContext.sort.collation.getFieldCollations() : null);
IndexSelector selector = new IndexSelector(indexCondition, remainderCondition, indexContext, collection, builder, totalRows);
for (IndexDescriptor indexDesc : collection) {
logger.info("index_plan_info indexDescriptor: {}", indexDesc.toString());
// check if any of the indexed fields of the index are present in the filter condition
if (IndexPlanUtils.conditionIndexed(indexableExprMarker, indexDesc) != IndexPlanUtils.ConditionIndexed.NONE) {
if (isValidIndexHint && !indexContext.indexHint.equals(indexDesc.getIndexName())) {
logger.info("index_plan_info: Index {} is being discarded due to index Hint", indexDesc.getIndexName());
continue;
}
FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo();
selector.addIndex(indexDesc, IndexPlanUtils.isCoveringIndex(indexContext, functionInfo), indexContext.lowerProject != null ? indexContext.lowerProject.getRowType().getFieldCount() : scan.getRowType().getFieldCount());
}
}
// get the candidate indexes based on selection
selector.getCandidateIndexes(infoBuilder, coveringIndexes, nonCoveringIndexes, intersectIndexes);
if (logger.isDebugEnabled()) {
StringBuilder strb = new StringBuilder();
if (coveringIndexes.size() > 0) {
strb.append("Covering indexes:");
for (IndexGroup index : coveringIndexes) {
strb.append(index.getIndexProps().get(0).getIndexDesc().getIndexName()).append(", ");
}
}
if (nonCoveringIndexes.size() > 0) {
strb.append("Non-covering indexes:");
for (IndexGroup index : nonCoveringIndexes) {
strb.append(index.getIndexProps().get(0).getIndexDesc().getIndexName()).append(", ");
}
}
logger.debug("index_plan_info: IndexSelector return: {}", strb.toString());
}
GroupScan primaryTableScan = indexContext.scan.getGroupScan();
// TODO: this logic for intersect should eventually be migrated to the IndexSelector
if (coveringIndexes.size() == 0 && nonCoveringIndexes.size() > 1) {
List<IndexDescriptor> indexList = Lists.newArrayList();
for (IndexGroup index : nonCoveringIndexes) {
IndexDescriptor indexDesc = index.getIndexProps().get(0).getIndexDesc();
IndexGroupScan idxScan = indexDesc.getIndexGroupScan();
// Copy primary table statistics to index table
idxScan.setStatistics(((DbGroupScan) primaryTableScan).getStatistics());
indexList.add(index.getIndexProps().get(0).getIndexDesc());
}
Map<IndexDescriptor, IndexConditionInfo> indexInfoMap = infoBuilder.getIndexConditionMap(indexList);
// no usable index
if (indexInfoMap == null || indexInfoMap.size() == 0) {
logger.info("index_plan_info: skipping intersect plan generation as there is no usable index");
return;
}
// some part of filter condition needs to apply on primary table.
if (indexInfoMap.size() > 1) {
logger.info("index_plan_info: intersect plan is generated");
if (logger.isDebugEnabled()) {
List<String> indices = new ArrayList<>(nonCoveringIndexes.size());
for (IndexGroup index : nonCoveringIndexes) {
indices.add(index.getIndexProps().get(0).getIndexDesc().getIndexName());
}
logger.debug("index_plan_info: intersect plan is generated on index list {}", indices);
}
boolean intersectPlanGenerated = false;
// TODO: make sure the smallest selectivity of these indexes times rowcount smaller than broadcast threshold
for (IndexGroup index : intersectIndexes) {
List<IndexDescriptor> candidateDesc = Lists.newArrayList();
for (IndexProperties candProp : index.getIndexProps()) {
candidateDesc.add(candProp.getIndexDesc());
}
Map<IndexDescriptor, IndexConditionInfo> intersectIdxInfoMap = infoBuilder.getIndexConditionMap(candidateDesc);
IndexIntersectPlanGenerator planGen = new IndexIntersectPlanGenerator(indexContext, intersectIdxInfoMap, builder, settings);
try {
planGen.go();
intersectPlanGenerated = true;
} catch (Exception e) {
// If error while generating intersect plans, continue onto generating non-covering plans
logger.warn("index_plan_info: Exception while trying to generate intersect index plan", e);
}
}
// If intersect plans are forced do not generate further non-covering plans
if (intersectPlanGenerated && settings.isIndexIntersectPlanPreferred()) {
return;
}
}
}
try {
for (IndexGroup index : coveringIndexes) {
IndexProperties indexProps = index.getIndexProps().get(0);
IndexDescriptor indexDesc = indexProps.getIndexDesc();
IndexGroupScan idxScan = indexDesc.getIndexGroupScan();
FunctionalIndexInfo indexInfo = indexDesc.getFunctionalInfo();
indexCondition = indexProps.getLeadingColumnsFilter();
remainderCondition = indexProps.getTotalRemainderFilter();
// Copy primary table statistics to index table
idxScan.setStatistics(((DbGroupScan) scan.getGroupScan()).getStatistics());
logger.info("index_plan_info: Generating covering index plan for index: {}, query condition {}", indexDesc.getIndexName(), indexCondition.toString());
CoveringIndexPlanGenerator planGen = new CoveringIndexPlanGenerator(indexContext, indexInfo, idxScan, indexCondition, remainderCondition, builder, settings);
planGen.go();
}
} catch (Exception e) {
logger.warn("Exception while trying to generate covering index plan", e);
}
// First, check if the primary table scan supports creating a restricted scan
if (primaryTableScan instanceof DbGroupScan && (((DbGroupScan) primaryTableScan).supportsRestrictedScan())) {
try {
for (IndexGroup index : nonCoveringIndexes) {
IndexProperties indexProps = index.getIndexProps().get(0);
IndexDescriptor indexDesc = indexProps.getIndexDesc();
IndexGroupScan idxScan = indexDesc.getIndexGroupScan();
indexCondition = indexProps.getLeadingColumnsFilter();
remainderCondition = indexProps.getTotalRemainderFilter();
// Copy primary table statistics to index table
idxScan.setStatistics(((DbGroupScan) primaryTableScan).getStatistics());
logger.info("index_plan_info: Generating non-covering index plan for index: {}, query condition {}", indexDesc.getIndexName(), indexCondition.toString());
NonCoveringIndexPlanGenerator planGen = new NonCoveringIndexPlanGenerator(indexContext, indexDesc, idxScan, indexCondition, remainderCondition, builder, settings);
planGen.go();
}
} catch (Exception e) {
logger.warn("Exception while trying to generate non-covering index access plan", e);
}
}
}
Aggregations