Examples with ImmutableBitSet org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet used on opensource projects

Example 66 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project flink by apache.

the class StreamPhysicalPythonOverAggregateRule method convert.

@Override
public RelNode convert(RelNode rel) {
    FlinkLogicalOverAggregate logicWindow = (FlinkLogicalOverAggregate) rel;
    if (logicWindow.groups.size() > 1) {
        throw new TableException("Over Agg: Unsupported use of OVER windows. " + "All aggregates must be computed on the same window. " + "please re-check the over window statement.");
    }
    ImmutableBitSet keys = logicWindow.groups.get(0).keys;
    FlinkRelDistribution requiredDistribution;
    if (!keys.isEmpty()) {
        requiredDistribution = FlinkRelDistribution.hash(keys.asList(), true);
    } else {
        requiredDistribution = FlinkRelDistribution.SINGLETON();
    }
    RelNode input = logicWindow.getInput();
    RelTraitSet requiredTraitSet = input.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL()).replace(requiredDistribution);
    RelTraitSet providedTraitSet = rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL());
    RelNode newInput = RelOptRule.convert(input, requiredTraitSet);
    return new StreamPhysicalPythonOverAggregate(rel.getCluster(), providedTraitSet, newInput, rel.getRowType(), logicWindow);
}

Example 67 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project flink by apache.

the class RankProcessStrategy method analyzeRankProcessStrategies.

/**
 * Gets {@link RankProcessStrategy} based on input, partitionKey and orderKey.
 */
static List<RankProcessStrategy> analyzeRankProcessStrategies(StreamPhysicalRel rank, ImmutableBitSet partitionKey, RelCollation orderKey) {
    FlinkRelMetadataQuery mq = (FlinkRelMetadataQuery) rank.getCluster().getMetadataQuery();
    List<RelFieldCollation> fieldCollations = orderKey.getFieldCollations();
    boolean isUpdateStream = !ChangelogPlanUtils.inputInsertOnly(rank);
    RelNode input = rank.getInput(0);
    if (isUpdateStream) {
        Set<ImmutableBitSet> upsertKeys = mq.getUpsertKeysInKeyGroupRange(input, partitionKey.toArray());
        if (upsertKeys == null || upsertKeys.isEmpty() || // upsert key should contains partition key
        upsertKeys.stream().noneMatch(k -> k.contains(partitionKey))) {
            // and we fall back to using retract rank
            return Collections.singletonList(RETRACT_STRATEGY);
        } else {
            FlinkRelMetadataQuery fmq = FlinkRelMetadataQuery.reuseOrCreate(mq);
            RelModifiedMonotonicity monotonicity = fmq.getRelModifiedMonotonicity(input);
            boolean isMonotonic = false;
            if (monotonicity != null && !fieldCollations.isEmpty()) {
                isMonotonic = fieldCollations.stream().allMatch(collation -> {
                    SqlMonotonicity fieldMonotonicity = monotonicity.fieldMonotonicities()[collation.getFieldIndex()];
                    RelFieldCollation.Direction direction = collation.direction;
                    if ((fieldMonotonicity == SqlMonotonicity.DECREASING || fieldMonotonicity == SqlMonotonicity.STRICTLY_DECREASING) && direction == RelFieldCollation.Direction.ASCENDING) {
                        // is decreasing
                        return true;
                    } else if ((fieldMonotonicity == SqlMonotonicity.INCREASING || fieldMonotonicity == SqlMonotonicity.STRICTLY_INCREASING) && direction == RelFieldCollation.Direction.DESCENDING) {
                        // is increasing
                        return true;
                    } else {
                        // it is monotonic
                        return fieldMonotonicity == SqlMonotonicity.CONSTANT;
                    }
                });
            }
            if (isMonotonic) {
                // TODO: choose a set of primary key
                return Arrays.asList(new UpdateFastStrategy(upsertKeys.iterator().next().toArray()), RETRACT_STRATEGY);
            } else {
                return Collections.singletonList(RETRACT_STRATEGY);
            }
        }
    } else {
        return Collections.singletonList(APPEND_FAST_STRATEGY);
    }
}

Example 68 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project beam by apache.

the class AggregateScanConverter method convert.

@Override
public RelNode convert(ResolvedAggregateScan zetaNode, List<RelNode> inputs) {
    LogicalProject input = convertAggregateScanInputScanToLogicalProject(zetaNode, inputs.get(0));
    // Calcite LogicalAggregate's GroupSet is indexes of group fields starting from 0.
    int groupFieldsListSize = zetaNode.getGroupByList().size();
    ImmutableBitSet groupSet;
    if (groupFieldsListSize != 0) {
        groupSet = ImmutableBitSet.of(IntStream.rangeClosed(0, groupFieldsListSize - 1).boxed().collect(Collectors.toList()));
    } else {
        groupSet = ImmutableBitSet.of();
    }
    // TODO: add support for indicator
    List<AggregateCall> aggregateCalls;
    if (zetaNode.getAggregateList().isEmpty()) {
        aggregateCalls = ImmutableList.of();
    } else {
        aggregateCalls = new ArrayList<>();
        // For aggregate calls, their input ref follow after GROUP BY input ref.
        int columnRefoff = groupFieldsListSize;
        for (ResolvedComputedColumn computedColumn : zetaNode.getAggregateList()) {
            AggregateCall aggCall = convertAggCall(computedColumn, columnRefoff, groupSet.size(), input);
            aggregateCalls.add(aggCall);
            if (!aggCall.getArgList().isEmpty()) {
                // Only increment column reference offset when aggregates use them (BEAM-8042).
                // Ex: COUNT(*) does not have arguments, while COUNT(`field`) does.
                columnRefoff++;
            }
        }
    }
    LogicalAggregate logicalAggregate = new LogicalAggregate(getCluster(), input.getTraitSet(), input, groupSet, ImmutableList.of(groupSet), aggregateCalls);
    return logicalAggregate;
}

Example 69 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project druid by druid-io.

the class Grouping method applyProject.

/**
 * Applies a post-grouping projection.
 *
 * @see DruidQuery#computeGrouping which uses this
 */
public Grouping applyProject(final PlannerContext plannerContext, final Project project) {
    final List<DimensionExpression> newDimensions = new ArrayList<>();
    final List<Aggregation> newAggregations = new ArrayList<>(aggregations);
    final Subtotals newSubtotals;
    final Projection postAggregationProjection = Projection.postAggregation(project, plannerContext, outputRowSignature, "p");
    postAggregationProjection.getPostAggregators().forEach(postAggregator -> newAggregations.add(Aggregation.create(postAggregator)));
    // Remove literal dimensions that did not appear in the projection. This is useful for queries
    // like "SELECT COUNT(*) FROM tbl GROUP BY 'dummy'" which some tools can generate, and for which we don't
    // actually want to include a dimension 'dummy'.
    final ImmutableBitSet aggregateProjectBits = RelOptUtil.InputFinder.bits(project.getChildExps(), null);
    final int[] newDimIndexes = new int[dimensions.size()];
    boolean droppedDimensions = false;
    for (int i = 0; i < dimensions.size(); i++) {
        final DimensionExpression dimension = dimensions.get(i);
        if (Parser.parse(dimension.getDruidExpression().getExpression(), plannerContext.getExprMacroTable()).isLiteral() && !aggregateProjectBits.get(i)) {
            droppedDimensions = true;
            newDimIndexes[i] = -1;
        } else {
            newDimIndexes[i] = newDimensions.size();
            newDimensions.add(dimension);
        }
    }
    // Renumber subtotals, if needed, to account for removed dummy dimensions.
    if (newDimensions.size() != dimensions.size()) {
        final List<IntList> newSubtotalsList = new ArrayList<>();
        for (IntList subtotal : subtotals.getSubtotals()) {
            final IntList newSubtotal = new IntArrayList();
            for (int dimIndex : subtotal) {
                final int newDimIndex = newDimIndexes[dimIndex];
                if (newDimIndex >= 0) {
                    newSubtotal.add(newDimIndex);
                }
            }
            newSubtotalsList.add(newSubtotal);
        }
        newSubtotals = new Subtotals(newSubtotalsList);
    } else {
        newSubtotals = subtotals;
    }
    return Grouping.create(newDimensions, newSubtotals, newAggregations, havingFilter, postAggregationProjection.getOutputRowSignature(), droppedDimensions);
}

Example 70 with ImmutableBitSet

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet in project hive by apache.

the class RelOptHiveTable method generateKeys.

private Pair<List<ImmutableBitSet>, List<ImmutableBitSet>> generateKeys() {
    final PrimaryKeyInfo primaryKeyInfo = hiveTblMetadata.getPrimaryKeyInfo();
    final UniqueConstraint uniqueKeyInfo = hiveTblMetadata.getUniqueKeyInfo();
    ImmutableList.Builder<ImmutableBitSet> builder = ImmutableList.builder();
    ImmutableList.Builder<ImmutableBitSet> nonNullbuilder = ImmutableList.builder();
    // First PK
    if (primaryKeyInfo != null && !primaryKeyInfo.getColNames().isEmpty()) {
        ImmutableBitSet.Builder keys = ImmutableBitSet.builder();
        for (String pkColName : primaryKeyInfo.getColNames().values()) {
            int pkPos;
            for (pkPos = 0; pkPos < rowType.getFieldNames().size(); pkPos++) {
                String colName = rowType.getFieldNames().get(pkPos);
                if (pkColName.equals(colName)) {
                    break;
                }
            }
            if (pkPos == rowType.getFieldNames().size()) {
                LOG.error("Column for primary key definition " + pkColName + " not found");
            }
            keys.set(pkPos);
        }
        ImmutableBitSet key = keys.build();
        builder.add(key);
        nonNullbuilder.add(key);
    }
    // Then UKs
    if (uniqueKeyInfo != null && !uniqueKeyInfo.getUniqueConstraints().isEmpty()) {
        for (List<UniqueConstraintCol> ukCols : uniqueKeyInfo.getUniqueConstraints().values()) {
            ImmutableBitSet.Builder keys = ImmutableBitSet.builder();
            boolean isNonNullable = true;
            for (UniqueConstraintCol ukCol : ukCols) {
                int ukPos;
                for (ukPos = 0; ukPos < rowType.getFieldNames().size(); ukPos++) {
                    String colName = rowType.getFieldNames().get(ukPos);
                    if (ukCol.colName.equals(colName)) {
                        if (rowType.getFieldList().get(ukPos).getType().isNullable()) {
                            // they should all be nullable
                            isNonNullable = false;
                        }
                        break;
                    }
                }
                if (ukPos == rowType.getFieldNames().size()) {
                    LOG.error("Column for unique constraint definition " + ukCol.colName + " not found");
                }
                keys.set(ukPos);
            }
            ImmutableBitSet key = keys.build();
            builder.add(key);
            if (isNonNullable) {
                nonNullbuilder.add(key);
            }
        }
    }
    return new Pair<>(builder.build(), nonNullbuilder.build());
}