Examples with RelTraitSet org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet used on opensource projects

Example 41 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by axbaretto.

the class SortConvertPrule method convert.

@Override
public RelNode convert(RelNode r) {
    Sort rel = (Sort) r;
    RelTraitSet traits = rel.getInput().getTraitSet().replace(Prel.DRILL_PHYSICAL);
    return new SortPrel(rel.getCluster(), traits.plus(rel.getCollation()), convert(rel.getInput(), traits.simplify()), rel.getCollation());
}

Example 42 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by axbaretto.

the class DirectScanPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    final DrillDirectScanRel scan = call.rel(0);
    final RelTraitSet traits = scan.getTraitSet().plus(Prel.DRILL_PHYSICAL);
    final ScanPrel newScan = new ScanPrel(scan.getCluster(), traits, scan.getGroupScan(), scan.getRowType()) {

        // direct scan (no execution) => no accidental column shuffling => no reordering
        @Override
        public boolean needsFinalColumnReordering() {
            return false;
        }
    };
    call.transformTo(newScan);
}

Example 43 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by axbaretto.

the class DefaultSqlHandler method convertToPrel.

/**
 * Applies physical rules and certain transformations to convert drill relational node into physical one.
 *
 * @param drel relational node
 * @param validatedRowType final output row type
 * @return physical relational node
 * @throws RelConversionException
 * @throws SqlUnsupportedException
 */
protected Prel convertToPrel(RelNode drel, RelDataType validatedRowType) throws RelConversionException, SqlUnsupportedException {
    Preconditions.checkArgument(drel.getConvention() == DrillRel.DRILL_LOGICAL);
    final RelTraitSet traits = drel.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
    Prel phyRelNode;
    try {
        final Stopwatch watch = Stopwatch.createStarted();
        final RelNode relNode = transform(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, drel, traits, false);
        phyRelNode = (Prel) relNode.accept(new PrelFinalizer());
        // log externally as we need to finalize before traversing the tree.
        log(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, phyRelNode, logger, watch);
    } catch (RelOptPlanner.CannotPlanException ex) {
        logger.error(ex.getMessage());
        if (JoinUtils.checkCartesianJoin(drel, new ArrayList<Integer>(), new ArrayList<Integer>(), new ArrayList<Boolean>())) {
            throw new UnsupportedRelOperatorException("This query cannot be planned possibly due to either a cartesian join or an inequality join");
        } else {
            throw ex;
        }
    }
    OptionManager queryOptions = context.getOptions();
    if (context.getPlannerSettings().isMemoryEstimationEnabled() && !MemoryEstimationVisitor.enoughMemory(phyRelNode, queryOptions, context.getActiveEndpoints().size())) {
        log("Not enough memory for this plan", phyRelNode, logger, null);
        logger.debug("Re-planning without hash operations.");
        queryOptions.setLocalOption(PlannerSettings.HASHJOIN.getOptionName(), false);
        queryOptions.setLocalOption(PlannerSettings.HASHAGG.getOptionName(), false);
        try {
            final RelNode relNode = transform(PlannerType.VOLCANO, PlannerPhase.PHYSICAL, drel, traits);
            phyRelNode = (Prel) relNode.accept(new PrelFinalizer());
        } catch (RelOptPlanner.CannotPlanException ex) {
            logger.error(ex.getMessage());
            if (JoinUtils.checkCartesianJoin(drel, new ArrayList<Integer>(), new ArrayList<Integer>(), new ArrayList<Boolean>())) {
                throw new UnsupportedRelOperatorException("This query cannot be planned possibly due to either a cartesian join or an inequality join");
            } else {
                throw ex;
            }
        }
    }
    /* The order of the following transformations is important */
    /*
     * 0.)
     * Add top project before screen operator or writer to ensure that final output column names are preserved.
     */
    phyRelNode = TopProjectVisitor.insertTopProject(phyRelNode, validatedRowType);
    /*
     * 1.) For select * from join query, we need insert project on top of scan and a top project just
     * under screen operator. The project on top of scan will rename from * to T1*, while the top project
     * will rename T1* to *, before it output the final result. Only the top project will allow
     * duplicate columns, since user could "explicitly" ask for duplicate columns ( select *, col, *).
     * The rest of projects will remove the duplicate column when we generate POP in json format.
     */
    phyRelNode = StarColumnConverter.insertRenameProject(phyRelNode);
    /*
     * 2.)
     * Join might cause naming conflicts from its left and right child.
     * In such case, we have to insert Project to rename the conflicting names.
     */
    phyRelNode = JoinPrelRenameVisitor.insertRenameProject(phyRelNode);
    /*
     * 2.1) Swap left / right for INNER hash join, if left's row count is < (1 + margin) right's row count.
     * We want to have smaller dataset on the right side, since hash table builds on right side.
     */
    if (context.getPlannerSettings().isHashJoinSwapEnabled()) {
        phyRelNode = SwapHashJoinVisitor.swapHashJoin(phyRelNode, Double.valueOf(context.getPlannerSettings().getHashJoinSwapMarginFactor()));
    }
    if (context.getPlannerSettings().isParquetRowGroupFilterPushdownPlanningEnabled()) {
        phyRelNode = (Prel) transform(PlannerType.HEP_BOTTOM_UP, PlannerPhase.PHYSICAL_PARTITION_PRUNING, phyRelNode);
    }
    /*
     * 2.2) Break up all expressions with complex outputs into their own project operations
     */
    phyRelNode = phyRelNode.accept(new SplitUpComplexExpressions(config.getConverter().getTypeFactory(), context.getDrillOperatorTable(), context.getPlannerSettings().functionImplementationRegistry), null);
    /*
     * 2.3) Projections that contain reference to flatten are rewritten as Flatten operators followed by Project
     */
    phyRelNode = phyRelNode.accept(new RewriteProjectToFlatten(config.getConverter().getTypeFactory(), context.getDrillOperatorTable()), null);
    /*
     * 3.)
     * Since our operators work via names rather than indices, we have to make to reorder any
     * output before we return data to the user as we may have accidentally shuffled things.
     * This adds a trivial project to reorder columns prior to output.
     */
    phyRelNode = FinalColumnReorderer.addFinalColumnOrdering(phyRelNode);
    /*
     * 4.)
     * If two fragments are both estimated to be parallelization one, remove the exchange
     * separating them
     */
    phyRelNode = ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode, targetSliceSize);
    /* 6.)
     * if the client does not support complex types (Map, Repeated)
     * insert a project which which would convert
     */
    if (!context.getSession().isSupportComplexTypes()) {
        logger.debug("Client does not support complex types, add ComplexToJson operator.");
        phyRelNode = ComplexToJsonPrelVisitor.addComplexToJsonPrel(phyRelNode);
    }
    /* 7.)
     * Insert LocalExchange (mux and/or demux) nodes
     */
    phyRelNode = InsertLocalExchangeVisitor.insertLocalExchanges(phyRelNode, queryOptions);
    /* 8.)
     * Next, we add any required selection vector removers given the supported encodings of each
     * operator. This will ultimately move to a new trait but we're managing here for now to avoid
     * introducing new issues in planning before the next release
     */
    phyRelNode = SelectionVectorPrelVisitor.addSelectionRemoversWhereNecessary(phyRelNode);
    /* 9.)
     * Finally, Make sure that the no rels are repeats.
     * This could happen in the case of querying the same table twice as Optiq may canonicalize these.
     */
    phyRelNode = RelUniqifier.uniqifyGraph(phyRelNode);
    return phyRelNode;
}

Example 44 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by axbaretto.

the class DefaultSqlHandler method transform.

/**
 * Transform RelNode to a new RelNode, targeting the provided set of traits. Also will log the outcome if asked.
 *
 * @param plannerType
 *          The type of Planner to use.
 * @param phase
 *          The transformation phase we're running.
 * @param input
 *          The origianl RelNode
 * @param targetTraits
 *          The traits we are targeting for output.
 * @param log
 *          Whether to log the planning phase.
 * @return The transformed relnode.
 */
protected RelNode transform(PlannerType plannerType, PlannerPhase phase, RelNode input, RelTraitSet targetTraits, boolean log) {
    final Stopwatch watch = Stopwatch.createStarted();
    final RuleSet rules = config.getRules(phase);
    final RelTraitSet toTraits = targetTraits.simplify();
    final RelNode output;
    switch(plannerType) {
        case HEP_BOTTOM_UP:
        case HEP:
            {
                final HepProgramBuilder hepPgmBldr = new HepProgramBuilder();
                if (plannerType == PlannerType.HEP_BOTTOM_UP) {
                    hepPgmBldr.addMatchOrder(HepMatchOrder.BOTTOM_UP);
                }
                for (RelOptRule rule : rules) {
                    hepPgmBldr.addRuleInstance(rule);
                }
                // Set noDAG = true to avoid caching problems which lead to incorrect Drill work.
                final HepPlanner planner = new HepPlanner(hepPgmBldr.build(), context.getPlannerSettings(), true, null, RelOptCostImpl.FACTORY);
                JaninoRelMetadataProvider relMetadataProvider = JaninoRelMetadataProvider.of(DrillDefaultRelMetadataProvider.INSTANCE);
                RelMetadataQuery.THREAD_PROVIDERS.set(relMetadataProvider);
                // Modify RelMetaProvider for every RelNode in the SQL operator Rel tree.
                input.accept(new MetaDataProviderModifier(relMetadataProvider));
                planner.setRoot(input);
                if (!input.getTraitSet().equals(targetTraits)) {
                    planner.changeTraits(input, toTraits);
                }
                output = planner.findBestExp();
                break;
            }
        case VOLCANO:
        default:
            {
                // as weird as it seems, the cluster's only planner is the volcano planner.
                final RelOptPlanner planner = input.getCluster().getPlanner();
                final Program program = Programs.of(rules);
                Preconditions.checkArgument(planner instanceof VolcanoPlanner, "Cluster is expected to be constructed using VolcanoPlanner. Was actually of type %s.", planner.getClass().getName());
                output = program.run(planner, input, toTraits, ImmutableList.<RelOptMaterialization>of(), ImmutableList.<RelOptLattice>of());
                break;
            }
    }
    if (log) {
        log(plannerType, phase, output, logger, watch);
    }
    return output;
}

Example 45 with RelTraitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet in project drill by axbaretto.

the class FindLimit0Visitor method getDirectScanRelIfFullySchemaed.

/**
 * If all field types of the given node are {@link #TYPES recognized types} and honored by execution, then this
 * method returns the tree: DrillDirectScanRel(field types). Otherwise, the method returns null.
 *
 * @param rel calcite logical rel tree
 * @return drill logical rel tree
 */
public static DrillRel getDirectScanRelIfFullySchemaed(RelNode rel) {
    final List<RelDataTypeField> fieldList = rel.getRowType().getFieldList();
    final List<TypeProtos.MajorType> columnTypes = Lists.newArrayList();
    for (final RelDataTypeField field : fieldList) {
        final SqlTypeName sqlTypeName = field.getType().getSqlTypeName();
        if (!TYPES.contains(sqlTypeName)) {
            return null;
        } else {
            final TypeProtos.MajorType.Builder builder = TypeProtos.MajorType.newBuilder().setMode(field.getType().isNullable() ? TypeProtos.DataMode.OPTIONAL : TypeProtos.DataMode.REQUIRED).setMinorType(TypeInferenceUtils.getDrillTypeFromCalciteType(sqlTypeName));
            if (TypeInferenceUtils.isScalarStringType(sqlTypeName)) {
                builder.setPrecision(field.getType().getPrecision());
            }
            columnTypes.add(builder.build());
        }
    }
    final RelTraitSet traits = rel.getTraitSet().plus(DrillRel.DRILL_LOGICAL);
    final RelDataTypeReader reader = new RelDataTypeReader(rel.getRowType().getFieldNames(), columnTypes);
    return new DrillDirectScanRel(rel.getCluster(), traits, new DirectGroupScan(reader, ScanStats.ZERO_RECORD_TABLE), rel.getRowType());
}