use of org.apache.drill.exec.planner.physical.Prel in project drill by apache.
the class ExcessiveExchangeIdentifier method visitScreen.
@Override
public Prel visitScreen(ScreenPrel prel, MajorFragmentStat s) throws RuntimeException {
s.addScreen(prel);
RelNode child = ((Prel) prel.getInput()).accept(this, s);
return (Prel) prel.copy(prel.getTraitSet(), Collections.singletonList(child));
}
use of org.apache.drill.exec.planner.physical.Prel in project drill by apache.
the class DefaultSqlHandler method convertToPrel.
protected Prel convertToPrel(RelNode drel) throws RelConversionException, SqlUnsupportedException {
Preconditions.checkArgument(drel.getConvention() == DrillRel.DRILL_LOGICAL);
final RelTraitSet traits = drel.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
Prel phyRelNode;
try {
final Stopwatch watch = Stopwatch.createStarted();
final RelNode relNode = transform(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, drel, traits, false);
phyRelNode = (Prel) relNode.accept(new PrelFinalizer());
// log externally as we need to finalize before traversing the tree.
log(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, phyRelNode, logger, watch);
} catch (RelOptPlanner.CannotPlanException ex) {
logger.error(ex.getMessage());
if (JoinUtils.checkCartesianJoin(drel, new ArrayList<Integer>(), new ArrayList<Integer>(), new ArrayList<Boolean>())) {
throw new UnsupportedRelOperatorException("This query cannot be planned possibly due to either a cartesian join or an inequality join");
} else {
throw ex;
}
}
OptionManager queryOptions = context.getOptions();
if (context.getPlannerSettings().isMemoryEstimationEnabled() && !MemoryEstimationVisitor.enoughMemory(phyRelNode, queryOptions, context.getActiveEndpoints().size())) {
log("Not enough memory for this plan", phyRelNode, logger, null);
logger.debug("Re-planning without hash operations.");
queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHJOIN.getOptionName(), false));
queryOptions.setOption(OptionValue.createBoolean(OptionValue.OptionType.QUERY, PlannerSettings.HASHAGG.getOptionName(), false));
try {
final RelNode relNode = transform(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, drel, traits);
phyRelNode = (Prel) relNode.accept(new PrelFinalizer());
} catch (RelOptPlanner.CannotPlanException ex) {
logger.error(ex.getMessage());
if (JoinUtils.checkCartesianJoin(drel, new ArrayList<Integer>(), new ArrayList<Integer>(), new ArrayList<Boolean>())) {
throw new UnsupportedRelOperatorException("This query cannot be planned possibly due to either a cartesian join or an inequality join");
} else {
throw ex;
}
}
}
/* The order of the following transformations is important */
/*
* 0.) For select * from join query, we need insert project on top of scan and a top project just
* under screen operator. The project on top of scan will rename from * to T1*, while the top project
* will rename T1* to *, before it output the final result. Only the top project will allow
* duplicate columns, since user could "explicitly" ask for duplicate columns ( select *, col, *).
* The rest of projects will remove the duplicate column when we generate POP in json format.
*/
phyRelNode = StarColumnConverter.insertRenameProject(phyRelNode);
/*
* 1.)
* Join might cause naming conflicts from its left and right child.
* In such case, we have to insert Project to rename the conflicting names.
*/
phyRelNode = JoinPrelRenameVisitor.insertRenameProject(phyRelNode);
/*
* 1.1) Swap left / right for INNER hash join, if left's row count is < (1 + margin) right's row count.
* We want to have smaller dataset on the right side, since hash table builds on right side.
*/
if (context.getPlannerSettings().isHashJoinSwapEnabled()) {
phyRelNode = SwapHashJoinVisitor.swapHashJoin(phyRelNode, new Double(context.getPlannerSettings().getHashJoinSwapMarginFactor()));
}
if (context.getPlannerSettings().isParquetRowGroupFilterPushdownPlanningEnabled()) {
phyRelNode = (Prel) transform(PlannerType.HEP_BOTTOM_UP, PlannerPhase.PHYSICAL_PARTITION_PRUNING, phyRelNode);
}
/*
* 1.2) Break up all expressions with complex outputs into their own project operations
*/
phyRelNode = phyRelNode.accept(new SplitUpComplexExpressions(config.getConverter().getTypeFactory(), context.getDrillOperatorTable(), context.getPlannerSettings().functionImplementationRegistry), null);
/*
* 1.3) Projections that contain reference to flatten are rewritten as Flatten operators followed by Project
*/
phyRelNode = phyRelNode.accept(new RewriteProjectToFlatten(config.getConverter().getTypeFactory(), context.getDrillOperatorTable()), null);
/*
* 2.)
* Since our operators work via names rather than indices, we have to make to reorder any
* output before we return data to the user as we may have accidentally shuffled things.
* This adds a trivial project to reorder columns prior to output.
*/
phyRelNode = FinalColumnReorderer.addFinalColumnOrdering(phyRelNode);
/*
* 3.)
* If two fragments are both estimated to be parallelization one, remove the exchange
* separating them
*/
phyRelNode = ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode, targetSliceSize);
/* 5.)
* if the client does not support complex types (Map, Repeated)
* insert a project which which would convert
*/
if (!context.getSession().isSupportComplexTypes()) {
logger.debug("Client does not support complex types, add ComplexToJson operator.");
phyRelNode = ComplexToJsonPrelVisitor.addComplexToJsonPrel(phyRelNode);
}
/* 6.)
* Insert LocalExchange (mux and/or demux) nodes
*/
phyRelNode = InsertLocalExchangeVisitor.insertLocalExchanges(phyRelNode, queryOptions);
/* 7.)
* Next, we add any required selection vector removers given the supported encodings of each
* operator. This will ultimately move to a new trait but we're managing here for now to avoid
* introducing new issues in planning before the next release
*/
phyRelNode = SelectionVectorPrelVisitor.addSelectionRemoversWhereNecessary(phyRelNode);
/* 8.)
* Finally, Make sure that the no rels are repeats.
* This could happen in the case of querying the same table twice as Optiq may canonicalize these.
*/
phyRelNode = RelUniqifier.uniqifyGraph(phyRelNode);
return phyRelNode;
}
use of org.apache.drill.exec.planner.physical.Prel in project drill by apache.
the class ExplainHandler method getPlan.
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException {
final ConvertedRelNode convertedRelNode = validateAndConvert(sqlNode);
final RelDataType validatedRowType = convertedRelNode.getValidatedRowType();
final RelNode queryRelNode = convertedRelNode.getConvertedNode();
log("Calcite", queryRelNode, logger, null);
DrillRel drel = convertToDrel(queryRelNode, validatedRowType);
if (mode == ResultMode.LOGICAL) {
LogicalExplain logicalResult = new LogicalExplain(drel, level, context);
return DirectPlan.createDirectPlan(context, logicalResult);
}
Prel prel = convertToPrel(drel);
logAndSetTextPlan("Drill Physical", prel, logger);
PhysicalOperator pop = convertToPop(prel);
PhysicalPlan plan = convertToPlan(pop);
log("Drill Plan", plan, logger);
PhysicalExplain physicalResult = new PhysicalExplain(prel, plan, level, context);
return DirectPlan.createDirectPlan(context, physicalResult);
}
use of org.apache.drill.exec.planner.physical.Prel in project drill by apache.
the class CreateTableHandler method getPlan.
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException {
SqlCreateTable sqlCreateTable = unwrap(sqlNode, SqlCreateTable.class);
String originalTableName = sqlCreateTable.getName();
final ConvertedRelNode convertedRelNode = validateAndConvert(sqlCreateTable.getQuery());
final RelDataType validatedRowType = convertedRelNode.getValidatedRowType();
final RelNode queryRelNode = convertedRelNode.getConvertedNode();
final RelNode newTblRelNode = SqlHandlerUtil.resolveNewTableRel(false, sqlCreateTable.getFieldNames(), validatedRowType, queryRelNode);
final DrillConfig drillConfig = context.getConfig();
final AbstractSchema drillSchema = resolveSchema(sqlCreateTable, config.getConverter().getDefaultSchema(), drillConfig);
checkDuplicatedObjectExistence(drillSchema, originalTableName, drillConfig, context.getSession());
final RelNode newTblRelNodeWithPCol = SqlHandlerUtil.qualifyPartitionCol(newTblRelNode, sqlCreateTable.getPartitionColumns());
log("Calcite", newTblRelNodeWithPCol, logger, null);
// Convert the query to Drill Logical plan and insert a writer operator on top.
StorageStrategy storageStrategy = sqlCreateTable.isTemporary() ? StorageStrategy.TEMPORARY : new StorageStrategy(context.getOption(ExecConstants.PERSISTENT_TABLE_UMASK).string_val, false);
// If we are creating temporary table, initial table name will be replaced with generated table name.
// Generated table name is unique, UUID.randomUUID() is used for its generation.
// Original table name is stored in temporary tables cache, so it can be substituted to generated one during querying.
String newTableName = sqlCreateTable.isTemporary() ? context.getSession().registerTemporaryTable(drillSchema, originalTableName, drillConfig) : originalTableName;
DrillRel drel = convertToDrel(newTblRelNodeWithPCol, drillSchema, newTableName, sqlCreateTable.getPartitionColumns(), newTblRelNode.getRowType(), storageStrategy);
Prel prel = convertToPrel(drel, newTblRelNode.getRowType(), sqlCreateTable.getPartitionColumns());
logAndSetTextPlan("Drill Physical", prel, logger);
PhysicalOperator pop = convertToPop(prel);
PhysicalPlan plan = convertToPlan(pop);
log("Drill Plan", plan, logger);
String message = String.format("Creating %s table [%s].", sqlCreateTable.isTemporary() ? "temporary" : "persistent", originalTableName);
logger.info(message);
return plan;
}
use of org.apache.drill.exec.planner.physical.Prel in project drill by apache.
the class SplitUpComplexExpressions method visitPrel.
@Override
public Prel visitPrel(Prel prel, Object value) throws RelConversionException {
List<RelNode> children = Lists.newArrayList();
for (Prel child : prel) {
child = child.accept(this, null);
children.add(child);
}
return (Prel) prel.copy(prel.getTraitSet(), children);
}
Aggregations