use of org.apache.drill.exec.planner.physical.DrillDistributionTrait in project drill by apache.
the class AbstractIndexPlanGenerator method getSortNode.
public static RelNode getSortNode(IndexCallContext indexContext, RelNode newRel, boolean donotGenerateSort, boolean isSingleton, boolean isExchangeRequired) {
OrderedRel rel = indexContext.getSort();
DrillDistributionTrait hashDistribution = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(indexContext.getDistributionFields()));
if (toRemoveSort(indexContext.getCollation(), newRel.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE))) {
// we are going to remove sort
logger.debug("Not generating SortPrel since we have the required collation");
if (IndexPlanUtils.generateLimit(rel)) {
newRel = new LimitPrel(newRel.getCluster(), newRel.getTraitSet().plus(indexContext.getCollation()).plus(Prel.DRILL_PHYSICAL), newRel, IndexPlanUtils.getOffset(rel), IndexPlanUtils.getFetch(rel));
}
RelTraitSet traits = newRel.getTraitSet().plus(indexContext.getCollation()).plus(Prel.DRILL_PHYSICAL);
newRel = Prule.convert(newRel, traits);
newRel = getExchange(newRel.getCluster(), isSingleton, isExchangeRequired, traits, hashDistribution, indexContext, newRel);
} else {
if (donotGenerateSort) {
logger.debug("Not generating SortPrel and index plan, since just picking index for full index scan is not beneficial.");
return null;
}
RelTraitSet traits = newRel.getTraitSet().plus(indexContext.getCollation()).plus(Prel.DRILL_PHYSICAL);
newRel = getSortOrTopN(indexContext, rel, newRel, Prule.convert(newRel, newRel.getTraitSet().replace(Prel.DRILL_PHYSICAL)));
newRel = getExchange(newRel.getCluster(), isSingleton, isExchangeRequired, traits, hashDistribution, indexContext, newRel);
}
return newRel;
}
use of org.apache.drill.exec.planner.physical.DrillDistributionTrait in project drill by apache.
the class IndexPlanUtils method buildCoveringIndexScan.
public static ScanPrel buildCoveringIndexScan(DrillScanRelBase origScan, IndexGroupScan indexGroupScan, IndexCallContext indexContext, IndexDescriptor indexDesc) {
FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo();
// to record the new (renamed)paths added
List<SchemaPath> rewrittenPaths = Lists.newArrayList();
DbGroupScan dbGroupScan = (DbGroupScan) getGroupScan(origScan);
indexGroupScan.setColumns(rewriteFunctionColumn(dbGroupScan.getColumns(), functionInfo, rewrittenPaths));
DrillDistributionTrait partition = scanIsPartition(getGroupScan(origScan)) ? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON;
RelDataType newRowType = FunctionalIndexHelper.rewriteFunctionalRowType(origScan, indexContext, functionInfo, rewrittenPaths);
// add a default collation trait otherwise Calcite runs into a ClassCastException, which at first glance
// seems like a Calcite bug
RelTraitSet indexScanTraitSet = origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(RelCollationTraitDef.INSTANCE.getDefault()).plus(partition);
// condition that the index actually has collation property (e.g hash indexes don't)
if (indexDesc.getCollation() != null) {
RelCollation collationTrait = buildCollationCoveringIndexScan(indexDesc, indexContext);
indexScanTraitSet = indexScanTraitSet.plus(collationTrait);
}
ScanPrel indexScanPrel = new ScanPrel(origScan.getCluster(), indexScanTraitSet, indexGroupScan, newRowType, origScan.getTable());
return indexScanPrel;
}
use of org.apache.drill.exec.planner.physical.DrillDistributionTrait in project drill by apache.
the class IndexIntersectPlanGenerator method buildRestrictedDBScan.
private Pair<RelNode, DbGroupScan> buildRestrictedDBScan(RexNode remnant, boolean isAnyIndexAsync) {
DbGroupScan origDbGroupScan = (DbGroupScan) IndexPlanUtils.getGroupScan(origScan);
List<SchemaPath> cols = new ArrayList<SchemaPath>(origDbGroupScan.getColumns());
if (!checkRowKey(cols)) {
cols.add(origDbGroupScan.getRowKeyPath());
}
// Create a restricted groupscan from the primary table's groupscan
DbGroupScan restrictedGroupScan = origDbGroupScan.getRestrictedScan(cols);
if (restrictedGroupScan == null) {
logger.error("Null restricted groupscan in IndexIntersectPlanGenerator.convertChild");
return null;
}
DrillDistributionTrait partition = IndexPlanUtils.scanIsPartition(IndexPlanUtils.getGroupScan(origScan)) ? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON;
RelNode lastRelNode;
RelDataType dbscanRowType = convertRowType(origScan.getRowType(), origScan.getCluster().getTypeFactory());
ScanPrel dbScan = new ScanPrel(origScan.getCluster(), origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(partition), restrictedGroupScan, dbscanRowType, origScan.getTable());
lastRelNode = dbScan;
// build the row type for the left Project
List<RexNode> leftProjectExprs = Lists.newArrayList();
int leftRowKeyIndex = getRowKeyIndex(dbScan.getRowType(), origScan);
final RelDataTypeField leftRowKeyField = dbScan.getRowType().getFieldList().get(leftRowKeyIndex);
final RelDataTypeFactory.FieldInfoBuilder leftFieldTypeBuilder = dbScan.getCluster().getTypeFactory().builder();
FilterPrel leftIndexFilterPrel = null;
// full original condition.
if (isAnyIndexAsync) {
new FilterPrel(dbScan.getCluster(), dbScan.getTraitSet(), dbScan, indexContext.getOrigCondition());
lastRelNode = leftIndexFilterPrel;
}
// new Project's rowtype is original Project's rowtype [plus rowkey if rowkey is not in original rowtype]
ProjectPrel leftIndexProjectPrel = null;
if (origProject != null) {
RelDataType origRowType = origProject.getRowType();
List<RelDataTypeField> origProjFields = origRowType.getFieldList();
leftFieldTypeBuilder.addAll(origProjFields);
// get the exprs from the original Project IFF there is a project
leftProjectExprs.addAll(IndexPlanUtils.getProjects(origProject));
// add the rowkey IFF rowkey is not in orig scan
if (getRowKeyIndex(origRowType, origScan) < 0) {
leftFieldTypeBuilder.add(leftRowKeyField);
leftProjectExprs.add(RexInputRef.of(leftRowKeyIndex, dbScan.getRowType()));
}
final RelDataType leftProjectRowType = leftFieldTypeBuilder.build();
leftIndexProjectPrel = new ProjectPrel(dbScan.getCluster(), dbScan.getTraitSet(), leftIndexFilterPrel == null ? dbScan : leftIndexFilterPrel, leftProjectExprs, leftProjectRowType);
lastRelNode = leftIndexProjectPrel;
}
final RelTraitSet leftTraits = dbScan.getTraitSet().plus(Prel.DRILL_PHYSICAL);
// final RelNode convertedLeft = convert(leftIndexProjectPrel, leftTraits);
final RelNode convertedLeft = Prule.convert(lastRelNode, leftTraits);
return new Pair<>(convertedLeft, restrictedGroupScan);
}
use of org.apache.drill.exec.planner.physical.DrillDistributionTrait in project drill by apache.
the class NonCoveringIndexPlanGenerator method convertChild.
@Override
public RelNode convertChild(final RelNode topRel, final RelNode input) throws InvalidRelException {
if (indexGroupScan == null) {
logger.error("Null indexgroupScan in NonCoveringIndexPlanGenerator.convertChild");
return null;
}
RelDataType dbscanRowType = convertRowType(origScan.getRowType(), origScan.getCluster().getTypeFactory());
RelDataType indexScanRowType = FunctionalIndexHelper.convertRowTypeForIndexScan(origScan, indexContext.getOrigMarker(), indexGroupScan, functionInfo);
DrillDistributionTrait partition = IndexPlanUtils.scanIsPartition(IndexPlanUtils.getGroupScan(origScan)) ? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON;
ScanPrel indexScanPrel = new ScanPrel(origScan.getCluster(), origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(partition), indexGroupScan, indexScanRowType, origScan.getTable());
DbGroupScan origDbGroupScan = (DbGroupScan) IndexPlanUtils.getGroupScan(origScan);
// right (build) side of the rowkey join: do a distribution of project-filter-indexscan subplan
RexNode convertedIndexCondition = FunctionalIndexHelper.convertConditionForIndexScan(indexCondition, origScan, indexScanRowType, builder, functionInfo);
FilterPrel rightIndexFilterPrel = new FilterPrel(indexScanPrel.getCluster(), indexScanPrel.getTraitSet(), indexScanPrel, convertedIndexCondition);
double finalRowCount = indexGroupScan.getRowCount(indexContext.getOrigCondition(), origScan);
// project the rowkey column from the index scan
List<RexNode> rightProjectExprs = Lists.newArrayList();
// indexGroupScan.getRowKeyOrdinal();
int rightRowKeyIndex = getRowKeyIndex(indexScanPrel.getRowType(), origScan);
assert rightRowKeyIndex >= 0;
rightProjectExprs.add(RexInputRef.of(rightRowKeyIndex, indexScanPrel.getRowType()));
final List<RelDataTypeField> indexScanFields = indexScanPrel.getRowType().getFieldList();
final RelDataTypeFactory.FieldInfoBuilder rightFieldTypeBuilder = indexScanPrel.getCluster().getTypeFactory().builder();
// build the row type for the right Project
final RelDataTypeField rightRowKeyField = indexScanFields.get(rightRowKeyIndex);
rightFieldTypeBuilder.add(rightRowKeyField);
final RelDataType rightProjectRowType = rightFieldTypeBuilder.build();
final ProjectPrel rightIndexProjectPrel = new ProjectPrel(indexScanPrel.getCluster(), indexScanPrel.getTraitSet(), rightIndexFilterPrel, rightProjectExprs, rightProjectRowType);
// create a RANGE PARTITION on the right side (this could be removed later during ExcessiveExchangeIdentifier phase
// if the estimated row count is smaller than slice_target
final RelNode rangeDistRight = createRangeDistRight(rightIndexProjectPrel, rightRowKeyField, origDbGroupScan);
// the range partitioning adds an extra column for the partition id but in the final plan we already have a
// renaming Project for the _id field inserted as part of the JoinPrelRenameVisitor. Thus, we are not inserting
// a separate Project here.
final RelNode convertedRight = rangeDistRight;
// left (probe) side of the rowkey join
List<SchemaPath> cols = new ArrayList<SchemaPath>(origDbGroupScan.getColumns());
if (!checkRowKey(cols)) {
cols.add(origDbGroupScan.getRowKeyPath());
}
// Create a restricted groupscan from the primary table's groupscan
DbGroupScan restrictedGroupScan = (DbGroupScan) origDbGroupScan.getRestrictedScan(cols);
if (restrictedGroupScan == null) {
logger.error("Null restricted groupscan in NonCoveringIndexPlanGenerator.convertChild");
return null;
}
// Set left side (restricted scan) row count as rows returned from right side (index scan)
DrillScanRel rightIdxRel = new DrillScanRel(origScan.getCluster(), origScan.getTraitSet(), origScan.getTable(), origScan.getRowType(), indexContext.getScanColumns());
double rightIdxRowCount = indexGroupScan.getRowCount(indexCondition, rightIdxRel);
restrictedGroupScan.setRowCount(null, rightIdxRowCount, rightIdxRowCount);
RelTraitSet origScanTraitSet = origScan.getTraitSet();
RelTraitSet restrictedScanTraitSet = origScanTraitSet.plus(Prel.DRILL_PHYSICAL);
// Create the collation traits for restricted scan based on the index columns under the
// conditions that (a) the index actually has collation property (e.g hash indexes don't)
// and (b) if an explicit sort operation is not enforced
RelCollation collation = null;
if (indexDesc.getCollation() != null && !settings.isIndexForceSortNonCovering()) {
collation = IndexPlanUtils.buildCollationNonCoveringIndexScan(indexDesc, indexScanRowType, dbscanRowType, indexContext);
if (restrictedScanTraitSet.contains(RelCollationTraitDef.INSTANCE)) {
// replace existing trait
restrictedScanTraitSet = restrictedScanTraitSet.plus(partition).replace(collation);
} else {
// add new one
restrictedScanTraitSet = restrictedScanTraitSet.plus(partition).plus(collation);
}
}
ScanPrel dbScan = new ScanPrel(origScan.getCluster(), restrictedScanTraitSet, restrictedGroupScan, dbscanRowType, origScan.getTable());
RelNode lastLeft = dbScan;
// build the row type for the left Project
List<RexNode> leftProjectExprs = Lists.newArrayList();
int leftRowKeyIndex = getRowKeyIndex(dbScan.getRowType(), origScan);
final RelDataTypeField leftRowKeyField = dbScan.getRowType().getFieldList().get(leftRowKeyIndex);
final RelDataTypeFactory.FieldInfoBuilder leftFieldTypeBuilder = dbScan.getCluster().getTypeFactory().builder();
// We are applying the same index condition to primary table's restricted scan. The reason is, the index may be an async
// index .. i.e it is not synchronously updated along with the primary table update as part of a single transaction, so it
// is possible that after or during index scan, the primary table rows may have been updated and no longer satisfy the index
// condition. By re-applying the index condition here, we will ensure non-qualifying records are filtered out.
// The remainder condition will be applied on top of RowKeyJoin.
FilterPrel leftIndexFilterPrel = null;
if (indexDesc.isAsyncIndex()) {
leftIndexFilterPrel = new FilterPrel(dbScan.getCluster(), dbScan.getTraitSet(), dbScan, indexContext.getOrigCondition());
lastLeft = leftIndexFilterPrel;
}
RelDataType origRowType = origProject == null ? origScan.getRowType() : origProject.getRowType();
if (origProject != null) {
// then we also don't need a project
// new Project's rowtype is original Project's rowtype [plus rowkey if rowkey is not in original rowtype]
List<RelDataTypeField> origProjFields = origRowType.getFieldList();
leftFieldTypeBuilder.addAll(origProjFields);
// get the exprs from the original Project
leftProjectExprs.addAll(IndexPlanUtils.getProjects(origProject));
// add the rowkey IFF rowkey is not in orig scan
if (getRowKeyIndex(origRowType, origScan) < 0) {
leftFieldTypeBuilder.add(leftRowKeyField);
leftProjectExprs.add(RexInputRef.of(leftRowKeyIndex, dbScan.getRowType()));
}
final RelDataType leftProjectRowType = leftFieldTypeBuilder.build();
// build collation in project
if (!settings.isIndexForceSortNonCovering()) {
collation = IndexPlanUtils.buildCollationProject(leftProjectExprs, null, dbScan, functionInfo, indexContext);
}
final ProjectPrel leftIndexProjectPrel = new ProjectPrel(dbScan.getCluster(), collation != null ? dbScan.getTraitSet().plus(collation) : dbScan.getTraitSet(), leftIndexFilterPrel == null ? dbScan : leftIndexFilterPrel, leftProjectExprs, leftProjectRowType);
lastLeft = leftIndexProjectPrel;
}
final RelTraitSet leftTraits = dbScan.getTraitSet().plus(Prel.DRILL_PHYSICAL);
// final RelNode convertedLeft = convert(leftIndexProjectPrel, leftTraits);
final RelNode convertedLeft = Prule.convert(lastLeft, leftTraits);
// find the rowkey column on the left side of join
final int leftRowKeyIdx = getRowKeyIndex(convertedLeft.getRowType(), origScan);
// only rowkey field is being projected from right side
final int rightRowKeyIdx = 0;
assert leftRowKeyIdx >= 0;
List<Integer> leftJoinKeys = ImmutableList.of(leftRowKeyIdx);
List<Integer> rightJoinKeys = ImmutableList.of(rightRowKeyIdx);
RexNode joinCondition = RelOptUtil.createEquiJoinCondition(convertedLeft, leftJoinKeys, convertedRight, rightJoinKeys, builder);
RelNode newRel;
if (settings.isIndexUseHashJoinNonCovering()) {
// for hash join, collation will be cleared
HashJoinPrel hjPrel = new HashJoinPrel(topRel.getCluster(), leftTraits, convertedLeft, convertedRight, joinCondition, JoinRelType.INNER, false, /* no swap */
null, /* no runtime filter */
true, /* useful for join-restricted scans */
JoinControl.DEFAULT);
newRel = hjPrel;
} else {
// if there is collation, add to rowkey join
RowKeyJoinPrel rjPrel = new RowKeyJoinPrel(topRel.getCluster(), collation != null ? leftTraits.plus(collation) : leftTraits, convertedLeft, convertedRight, joinCondition, JoinRelType.INNER);
rjPrel.setEstimatedRowCount(finalRowCount);
newRel = rjPrel;
}
final RelDataTypeFactory.FieldInfoBuilder finalFieldTypeBuilder = origScan.getCluster().getTypeFactory().builder();
List<RelDataTypeField> rjRowFields = newRel.getRowType().getFieldList();
int toRemoveRowKeyCount = 1;
if (getRowKeyIndex(origRowType, origScan) < 0) {
toRemoveRowKeyCount = 2;
}
finalFieldTypeBuilder.addAll(rjRowFields.subList(0, rjRowFields.size() - toRemoveRowKeyCount));
final RelDataType finalProjectRowType = finalFieldTypeBuilder.build();
List<RexNode> resetExprs = Lists.newArrayList();
for (int idx = 0; idx < rjRowFields.size() - toRemoveRowKeyCount; ++idx) {
resetExprs.add(RexInputRef.of(idx, newRel.getRowType()));
}
// rewrite the collation for this projectPrel
final ProjectPrel resetProjectPrel = new ProjectPrel(newRel.getCluster(), newRel.getTraitSet(), newRel, resetExprs, finalProjectRowType);
newRel = resetProjectPrel;
if (upperProject != null) {
RelCollation newCollation = RelCollations.of(RelCollations.EMPTY.getFieldCollations());
DrillDistributionTrait newDist = null;
newDist = upperProject.getInput().getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE);
if (!settings.isIndexForceSortNonCovering()) {
newCollation = IndexPlanUtils.buildCollationProject(IndexPlanUtils.getProjects(upperProject), origProject, origScan, functionInfo, indexContext);
}
RelTraitSet newProjectTraits = newTraitSet(Prel.DRILL_PHYSICAL, newDist, newCollation);
ProjectPrel cap = new ProjectPrel(upperProject.getCluster(), newProjectTraits, newRel, IndexPlanUtils.getProjects(upperProject), upperProject.getRowType());
newRel = cap;
}
// whether to remove sort
if (indexContext.getSort() != null) {
// unsorted input because ordering is not guaranteed across different parallel inputs.
if (toRemoveSort(indexContext.getCollation(), newRel.getTraitSet().getTrait(RelCollationTraitDef.INSTANCE))) {
((IndexGroupScan) indexScanPrel.getGroupScan()).setParallelizationWidth(1);
}
newRel = getSortNode(indexContext, newRel, false, true, true);
Preconditions.checkArgument(newRel != null);
}
RelNode finalRel = Prule.convert(newRel, newRel.getTraitSet());
logger.debug("NonCoveringIndexPlanGenerator got finalRel {} from origScan {}", finalRel.toString(), origScan.toString());
return finalRel;
}
use of org.apache.drill.exec.planner.physical.DrillDistributionTrait in project drill by apache.
the class IndexIntersectPlanGenerator method buildIntersectPlan.
public RelNode buildIntersectPlan(Map.Entry<IndexDescriptor, RexNode> pair, RelNode right, boolean generateDistribution) throws InvalidRelException {
IndexDescriptor index = pair.getKey();
RexNode condition = pair.getValue();
FunctionalIndexInfo functionInfo = getFunctionalIndexInfo(index);
IndexGroupScan indexScan = index.getIndexGroupScan();
RelDataType indexScanRowType = FunctionalIndexHelper.convertRowTypeForIndexScan(origScan, indexContext.getOrigMarker(), indexScan, functionInfo);
DrillDistributionTrait partition = IndexPlanUtils.scanIsPartition(IndexPlanUtils.getGroupScan(origScan)) ? DrillDistributionTrait.RANDOM_DISTRIBUTED : DrillDistributionTrait.SINGLETON;
ScanPrel indexScanPrel = new ScanPrel(origScan.getCluster(), origScan.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(partition), indexScan, indexScanRowType, origScan.getTable());
FilterPrel indexFilterPrel = new FilterPrel(indexScanPrel.getCluster(), indexScanPrel.getTraitSet(), indexScanPrel, FunctionalIndexHelper.convertConditionForIndexScan(condition, origScan, indexScanRowType, builder, functionInfo));
// project the rowkey column from the index scan
List<RexNode> indexProjectExprs = Lists.newArrayList();
int rowKeyIndex = getRowKeyIndex(indexScanPrel.getRowType(), origScan);
assert rowKeyIndex >= 0;
indexProjectExprs.add(RexInputRef.of(rowKeyIndex, indexScanPrel.getRowType()));
final RelDataTypeFactory.FieldInfoBuilder rightFieldTypeBuilder = indexScanPrel.getCluster().getTypeFactory().builder();
// build the row type for the right Project
final List<RelDataTypeField> indexScanFields = indexScanPrel.getRowType().getFieldList();
final RelDataTypeField rightRowKeyField = indexScanFields.get(rowKeyIndex);
rightFieldTypeBuilder.add(rightRowKeyField);
final RelDataType indexProjectRowType = rightFieldTypeBuilder.build();
final ProjectPrel indexProjectPrel = new ProjectPrel(indexScanPrel.getCluster(), indexScanPrel.getTraitSet(), indexFilterPrel, indexProjectExprs, indexProjectRowType);
RelTraitSet rightSideTraits = newTraitSet().plus(Prel.DRILL_PHYSICAL);
// if build(right) side does not exist, this index scan is the right most.
if (right == null) {
if (partition == DrillDistributionTrait.RANDOM_DISTRIBUTED && settings.getSliceTarget() < indexProjectPrel.getRows()) {
final DrillDistributionTrait distRight = new DrillDistributionTrait(DistributionType.BROADCAST_DISTRIBUTED);
rightSideTraits = newTraitSet(distRight).plus(Prel.DRILL_PHYSICAL);
}
}
RelNode converted = Prule.convert(indexProjectPrel, rightSideTraits);
if (right == null) {
return converted;
}
// if build(right) side exist, the plan we got in 'converted' is left (probe). Intersect with right(build) side
RelNode finalRel = buildRowKeyJoin(converted, right, false, JoinControl.INTERSECT_DISTINCT);
if (generateDistribution && right.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE) != DrillDistributionTrait.SINGLETON) {
final DrillDistributionTrait distRight = new DrillDistributionTrait(DistributionType.BROADCAST_DISTRIBUTED);
rightSideTraits = newTraitSet(distRight).plus(Prel.DRILL_PHYSICAL);
// This join will serve as the right side for the next intersection join, if applicable
finalRel = Prule.convert(finalRel, rightSideTraits);
}
logger.trace("IndexIntersectPlanGenerator got finalRel {} from origScan {}", finalRel.toString(), origScan.toString());
return finalRel;
}
Aggregations