Examples with Project org.apache.calcite.rel.core.Project used on opensource projects

Example 81 with Project

use of org.apache.calcite.rel.core.Project in project hive by apache.

the class RelFieldTrimmer method trimFields.

/**
 * Variant of {@link #trimFields(RelNode, ImmutableBitSet, Set)} for
 * {@link org.apache.calcite.rel.logical.LogicalTableScan}.
 */
public TrimResult trimFields(final TableScan tableAccessRel, ImmutableBitSet fieldsUsed, Set<RelDataTypeField> extraFields) {
    final int fieldCount = tableAccessRel.getRowType().getFieldCount();
    if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount)) && extraFields.isEmpty()) {
        // then no need to introduce another RelNode
        return trimFields((RelNode) tableAccessRel, fieldsUsed, extraFields);
    }
    final RelNode newTableAccessRel = tableAccessRel.project(fieldsUsed, extraFields, REL_BUILDER.get());
    // pretend that one field is used.
    if (fieldsUsed.cardinality() == 0) {
        RelNode input = newTableAccessRel;
        if (input instanceof Project) {
            // The table has implemented the project in the obvious way - by
            // creating project with 0 fields. Strip it away, and create our own
            // project with one field.
            Project project = (Project) input;
            if (project.getRowType().getFieldCount() == 0) {
                input = project.getInput();
            }
        }
        return dummyProject(fieldCount, input);
    }
    final Mapping mapping = createMapping(fieldsUsed, fieldCount);
    return result(newTableAccessRel, mapping);
}

Example 82 with Project

use of org.apache.calcite.rel.core.Project in project hive by apache.

the class HiveAggregateJoinTransposeRule method isGroupingUnique.

/**
 * Determines weather the give grouping is unique.
 *
 * Consider a join which might produce non-unique rows; but later the results are aggregated again.
 * This method determines if there are sufficient columns in the grouping which have been present previously as unique column(s).
 */
private boolean isGroupingUnique(RelNode input, ImmutableBitSet groups) {
    if (groups.isEmpty()) {
        return false;
    }
    if (input instanceof HepRelVertex) {
        HepRelVertex vertex = (HepRelVertex) input;
        return isGroupingUnique(vertex.getCurrentRel(), groups);
    }
    RelMetadataQuery mq = input.getCluster().getMetadataQuery();
    Set<ImmutableBitSet> uKeys = mq.getUniqueKeys(input);
    if (uKeys == null) {
        return false;
    }
    for (ImmutableBitSet u : uKeys) {
        if (groups.contains(u)) {
            return true;
        }
    }
    if (input instanceof Join) {
        Join join = (Join) input;
        JoinInfo ji = JoinInfo.of(join.getLeft(), join.getRight(), join.getCondition());
        if (ji.isEqui()) {
            ImmutableBitSet newGroup = groups.intersect(InputFinder.bits(join.getCondition()));
            RelNode l = join.getLeft();
            RelNode r = join.getRight();
            int joinFieldCount = join.getRowType().getFieldCount();
            int lFieldCount = l.getRowType().getFieldCount();
            ImmutableBitSet groupL = newGroup.get(0, lFieldCount);
            ImmutableBitSet groupR = newGroup.get(lFieldCount, joinFieldCount).shift(-lFieldCount);
            if (isGroupingUnique(l, groupL)) {
                return true;
            }
            if (isGroupingUnique(r, groupR)) {
                return true;
            }
        }
    }
    if (input instanceof Project) {
        Project project = (Project) input;
        ImmutableBitSet.Builder newGroup = ImmutableBitSet.builder();
        for (int g : groups.asList()) {
            RexNode rex = project.getProjects().get(g);
            if (rex instanceof RexInputRef) {
                RexInputRef rexInputRef = (RexInputRef) rex;
                newGroup.set(rexInputRef.getIndex());
            }
        }
        return isGroupingUnique(project.getInput(), newGroup.build());
    }
    return false;
}

Example 83 with Project

use of org.apache.calcite.rel.core.Project in project hive by apache.

the class HiveWindowingFixRule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    Project project = call.rel(0);
    Aggregate aggregate = call.rel(1);
    // 1. We go over the expressions in the project operator
    // and we separate the windowing nodes that are result
    // of an aggregate expression from the rest of nodes
    final int groupingFields = aggregate.getGroupCount() + aggregate.getIndicatorCount();
    Set<String> projectExprsDigest = new HashSet<String>();
    Map<String, RexNode> windowingExprsDigestToNodes = new HashMap<String, RexNode>();
    for (RexNode r : project.getProjects()) {
        if (r instanceof RexOver) {
            RexOver rexOverNode = (RexOver) r;
            // Operands
            for (RexNode operand : rexOverNode.getOperands()) {
                if (operand instanceof RexInputRef && ((RexInputRef) operand).getIndex() >= groupingFields) {
                    windowingExprsDigestToNodes.put(operand.toString(), operand);
                }
            }
            // Partition keys
            for (RexNode partitionKey : rexOverNode.getWindow().partitionKeys) {
                if (partitionKey instanceof RexInputRef && ((RexInputRef) partitionKey).getIndex() >= groupingFields) {
                    windowingExprsDigestToNodes.put(partitionKey.toString(), partitionKey);
                }
            }
            // Order keys
            for (RexFieldCollation orderKey : rexOverNode.getWindow().orderKeys) {
                if (orderKey.left instanceof RexInputRef && ((RexInputRef) orderKey.left).getIndex() >= groupingFields) {
                    windowingExprsDigestToNodes.put(orderKey.left.toString(), orderKey.left);
                }
            }
        } else {
            projectExprsDigest.add(r.toString());
        }
    }
    // 2. We check whether there is a column needed by the
    // windowing operation that is missing in the
    // project expressions. For instance, if the windowing
    // operation is over an aggregation column, Hive expects
    // that column to be in the Select clause of the query.
    // The idea is that if there is a column missing, we will
    // replace the old project operator by two new project
    // operators:
    // - a project operator containing the original columns
    // of the project operator plus all the columns that were
    // missing
    // - a project on top of the previous one, that will take
    // out the columns that were missing and were added by the
    // previous project
    // These data structures are needed to create the new project
    // operator (below)
    final List<RexNode> belowProjectExprs = new ArrayList<RexNode>();
    final List<String> belowProjectColumnNames = new ArrayList<String>();
    // This data structure is needed to create the new project
    // operator (top)
    final List<RexNode> topProjectExprs = new ArrayList<RexNode>();
    final int projectCount = project.getProjects().size();
    for (int i = 0; i < projectCount; i++) {
        belowProjectExprs.add(project.getProjects().get(i));
        belowProjectColumnNames.add(project.getRowType().getFieldNames().get(i));
        topProjectExprs.add(RexInputRef.of(i, project.getRowType()));
    }
    boolean windowingFix = false;
    for (Entry<String, RexNode> windowingExpr : windowingExprsDigestToNodes.entrySet()) {
        if (!projectExprsDigest.contains(windowingExpr.getKey())) {
            windowingFix = true;
            belowProjectExprs.add(windowingExpr.getValue());
            int colIndex = 0;
            String alias = "window_col_" + colIndex;
            while (belowProjectColumnNames.contains(alias)) {
                alias = "window_col_" + (colIndex++);
            }
            belowProjectColumnNames.add(alias);
        }
    }
    if (!windowingFix) {
        // We do not need to do anything, we bail out
        return;
    }
    // 3. We need to fix it, we create the two replacement project
    // operators
    RelNode newProjectRel = projectFactory.createProject(aggregate, Collections.emptyList(), belowProjectExprs, belowProjectColumnNames);
    RelNode newTopProjectRel = projectFactory.createProject(newProjectRel, Collections.emptyList(), topProjectExprs, project.getRowType().getFieldNames());
    call.transformTo(newTopProjectRel);
}

Example 84 with Project

use of org.apache.calcite.rel.core.Project in project hive by apache.

the class HiveRelFieldTrimmer method rewriteGBConstantKeys.

/**
 * This method replaces group by 'constant key' with group by true (boolean)
 * if and only if
 *  group by doesn't have grouping sets
 *  all keys in group by are constant
 *  none of the relnode above aggregate refers to these keys
 *
 *  If all of above is true then group by is rewritten and a new project is introduced
 *  underneath aggregate
 *
 *  This is mainly done so that hive is able to push down queries with
 *  group by 'constant key with type not supported by druid' into druid.
 */
private Aggregate rewriteGBConstantKeys(Aggregate aggregate, ImmutableBitSet fieldsUsed, ImmutableBitSet aggCallFields) {
    if ((aggregate.getIndicatorCount() > 0) || (aggregate.getGroupSet().isEmpty()) || fieldsUsed.contains(aggregate.getGroupSet())) {
        return aggregate;
    }
    final RelNode input = aggregate.getInput();
    final RelDataType rowType = input.getRowType();
    RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder();
    final List<RexNode> newProjects = new ArrayList<>();
    final List<RexNode> inputExprs = input instanceof Project ? ((Project) input).getProjects() : null;
    if (inputExprs == null || inputExprs.isEmpty()) {
        return aggregate;
    }
    boolean allConstants = true;
    for (int key : aggregate.getGroupSet()) {
        // getChildExprs on Join could return less number of expressions than there are coming out of join
        if (inputExprs.size() <= key || !isRexLiteral(inputExprs.get(key))) {
            allConstants = false;
            break;
        }
    }
    if (allConstants) {
        for (int i = 0; i < rowType.getFieldCount(); i++) {
            if (aggregate.getGroupSet().get(i) && !aggCallFields.get(i)) {
                newProjects.add(rexBuilder.makeLiteral(true));
            } else {
                newProjects.add(rexBuilder.makeInputRef(input, i));
            }
        }
        final RelBuilder relBuilder = REL_BUILDER.get();
        relBuilder.push(input);
        relBuilder.project(newProjects);
        Aggregate newAggregate = new HiveAggregate(aggregate.getCluster(), aggregate.getTraitSet(), relBuilder.build(), aggregate.getGroupSet(), aggregate.getGroupSets(), aggregate.getAggCallList());
        return newAggregate;
    }
    return aggregate;
}

Example 85 with Project

use of org.apache.calcite.rel.core.Project in project hive by apache.

the class HiveRelFieldTrimmer method fetchColStats.

private void fetchColStats(RelNode key, TableScan tableAccessRel, ImmutableBitSet fieldsUsed, Set<RelDataTypeField> extraFields) {
    final List<Integer> iRefSet = Lists.newArrayList();
    if (key instanceof Project) {
        final Project project = (Project) key;
        for (RexNode rx : project.getProjects()) {
            iRefSet.addAll(HiveCalciteUtil.getInputRefs(rx));
        }
    } else {
        final int fieldCount = tableAccessRel.getRowType().getFieldCount();
        if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount)) && extraFields.isEmpty()) {
            // get all cols
            iRefSet.addAll(ImmutableBitSet.range(fieldCount).asList());
        }
    }
    // Remove any virtual cols
    if (tableAccessRel instanceof HiveTableScan) {
        iRefSet.removeAll(((HiveTableScan) tableAccessRel).getVirtualCols());
    }
    if (!iRefSet.isEmpty()) {
        final RelOptTable table = tableAccessRel.getTable();
        if (table instanceof RelOptHiveTable) {
            ((RelOptHiveTable) table).getColStat(iRefSet, true);
            LOG.debug("Got col stats for {} in {}", iRefSet, tableAccessRel.getTable().getQualifiedName());
        }
    }
}