Examples with MetadataAggRel org.apache.drill.exec.planner.logical.MetadataAggRel used on opensource projects

Example 1 with MetadataAggRel

use of org.apache.drill.exec.planner.logical.MetadataAggRel in project drill by apache.

the class MetastoreAnalyzeTableHandler method getTableAggRelNode.

private DrillRel getTableAggRelNode(DrillRel convertedRelNode, boolean createNewAggregations, List<SchemaPath> statisticsColumns, SchemaPath locationField, MetadataHandlerContext handlerContext) {
    SchemaPath lastModifiedTimeField = SchemaPath.getSimplePath(config.getContext().getOptions().getString(ExecConstants.IMPLICIT_LAST_MODIFIED_TIME_COLUMN_LABEL));
    List<SchemaPath> metadataColumns = Arrays.asList(locationField, lastModifiedTimeField);
    MetadataAggregateContext aggregateContext = MetadataAggregateContext.builder().groupByExpressions(Collections.emptyList()).interestingColumns(statisticsColumns).createNewAggregations(createNewAggregations).metadataColumns(metadataColumns).metadataLevel(MetadataType.TABLE).build();
    convertedRelNode = new MetadataAggRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode, aggregateContext);
    convertedRelNode = new MetadataHandlerRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode, handlerContext);
    return convertedRelNode;
}

Example 2 with MetadataAggRel

use of org.apache.drill.exec.planner.logical.MetadataAggRel in project drill by apache.

the class MetastoreAnalyzeTableHandler method getSegmentAggRelNode.

private DrillRel getSegmentAggRelNode(List<NamedExpression> segmentExpressions, DrillRel convertedRelNode, boolean createNewAggregations, List<SchemaPath> statisticsColumns, SchemaPath locationField, int segmentLevel, MetadataHandlerContext handlerContext) {
    SchemaPath lastModifiedTimeField = SchemaPath.getSimplePath(config.getContext().getOptions().getString(ExecConstants.IMPLICIT_LAST_MODIFIED_TIME_COLUMN_LABEL));
    List<SchemaPath> metadataColumns = Arrays.asList(lastModifiedTimeField, locationField);
    List<NamedExpression> groupByExpressions = new ArrayList<>(segmentExpressions);
    MetadataAggregateContext aggregateContext = MetadataAggregateContext.builder().groupByExpressions(groupByExpressions.subList(0, segmentLevel)).interestingColumns(statisticsColumns).createNewAggregations(createNewAggregations).metadataColumns(metadataColumns).metadataLevel(MetadataType.SEGMENT).build();
    convertedRelNode = new MetadataAggRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode, aggregateContext);
    convertedRelNode = new MetadataHandlerRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode, handlerContext);
    return convertedRelNode;
}

Example 3 with MetadataAggRel

use of org.apache.drill.exec.planner.logical.MetadataAggRel in project drill by apache.

the class MetastoreAnalyzeTableHandler method getFileAggRelNode.

private DrillRel getFileAggRelNode(List<NamedExpression> segmentExpressions, DrillRel convertedRelNode, boolean createNewAggregations, List<SchemaPath> statisticsColumns, SchemaPath locationField, MetadataHandlerContext handlerContext) {
    SchemaPath lastModifiedTimeField = SchemaPath.getSimplePath(config.getContext().getOptions().getString(ExecConstants.IMPLICIT_LAST_MODIFIED_TIME_COLUMN_LABEL));
    List<SchemaPath> metadataColumns = Arrays.asList(lastModifiedTimeField, locationField);
    NamedExpression locationExpression = new NamedExpression(locationField, FieldReference.getWithQuotedRef(MetastoreAnalyzeConstants.LOCATION_FIELD));
    List<NamedExpression> fileGroupByExpressions = new ArrayList<>(segmentExpressions);
    fileGroupByExpressions.add(locationExpression);
    MetadataAggregateContext aggregateContext = MetadataAggregateContext.builder().groupByExpressions(fileGroupByExpressions).interestingColumns(statisticsColumns).createNewAggregations(createNewAggregations).metadataColumns(metadataColumns).metadataLevel(MetadataType.FILE).build();
    convertedRelNode = new MetadataAggRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode, aggregateContext);
    convertedRelNode = new MetadataHandlerRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode, handlerContext);
    return convertedRelNode;
}

Example 4 with MetadataAggRel

use of org.apache.drill.exec.planner.logical.MetadataAggRel in project drill by apache.

the class MetadataAggPrule method onMatch.

@Override
public void onMatch(RelOptRuleCall call) {
    MetadataAggRel aggregate = call.rel(0);
    RelNode input = aggregate.getInput();
    int groupByExprsSize = aggregate.getContext().groupByExpressions().size();
    List<RelFieldCollation> collations = new ArrayList<>();
    List<String> names = new ArrayList<>();
    for (int i = 0; i < groupByExprsSize; i++) {
        collations.add(new RelFieldCollation(i + 1));
        SchemaPath fieldPath = getArgumentReference(aggregate.getContext().groupByExpressions().get(i));
        names.add(fieldPath.getRootSegmentPath());
    }
    RelCollation collation = new NamedRelCollation(collations, names);
    RelTraitSet traits;
    if (aggregate.getContext().groupByExpressions().isEmpty()) {
        DrillDistributionTrait singleDist = DrillDistributionTrait.SINGLETON;
        RelTraitSet singleDistTrait = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(singleDist);
        createTransformRequest(call, aggregate, input, singleDistTrait);
    } else {
        // hash distribute on all grouping keys
        DrillDistributionTrait distOnAllKeys = new DrillDistributionTrait(DrillDistributionTrait.DistributionType.HASH_DISTRIBUTED, ImmutableList.copyOf(getDistributionFields(aggregate.getContext().groupByExpressions())));
        PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner());
        boolean smallInput = input.estimateRowCount(input.getCluster().getMetadataQuery()) < settings.getSliceTarget();
        // to produce sort locally before aggregation is produced for large inputs
        if (aggregate.getContext().createNewAggregations() && !smallInput) {
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL);
            RelNode convertedInput = convert(input, traits);
            new TwoPhaseMetadataAggSubsetTransformer(call, collation, distOnAllKeys).go(aggregate, convertedInput);
        } else {
            // TODO: DRILL-7433 - replace DrillDistributionTrait.SINGLETON with distOnAllKeys when parallelization for MetadataHandler is implemented
            traits = call.getPlanner().emptyTraitSet().plus(Prel.DRILL_PHYSICAL).plus(collation).plus(DrillDistributionTrait.SINGLETON);
            createTransformRequest(call, aggregate, input, traits);
        }
    }
}

Example 5 with MetadataAggRel

use of org.apache.drill.exec.planner.logical.MetadataAggRel in project drill by apache.

the class MetastoreAnalyzeTableHandler method getRowGroupAggRelNode.

private DrillRel getRowGroupAggRelNode(List<NamedExpression> segmentExpressions, DrillRel convertedRelNode, boolean createNewAggregations, List<SchemaPath> statisticsColumns, MetadataHandlerContext handlerContext) {
    SchemaPath locationField = SchemaPath.getSimplePath(config.getContext().getOptions().getString(ExecConstants.IMPLICIT_FQN_COLUMN_LABEL));
    SchemaPath lastModifiedTimeField = SchemaPath.getSimplePath(config.getContext().getOptions().getString(ExecConstants.IMPLICIT_LAST_MODIFIED_TIME_COLUMN_LABEL));
    String rowGroupIndexColumn = config.getContext().getOptions().getString(ExecConstants.IMPLICIT_ROW_GROUP_INDEX_COLUMN_LABEL);
    SchemaPath rgiField = SchemaPath.getSimplePath(rowGroupIndexColumn);
    List<NamedExpression> rowGroupGroupByExpressions = getRowGroupExpressions(segmentExpressions, locationField, rowGroupIndexColumn, rgiField);
    SchemaPath rowGroupStartField = SchemaPath.getSimplePath(config.getContext().getOptions().getString(ExecConstants.IMPLICIT_ROW_GROUP_START_COLUMN_LABEL));
    SchemaPath rowGroupLengthField = SchemaPath.getSimplePath(config.getContext().getOptions().getString(ExecConstants.IMPLICIT_ROW_GROUP_LENGTH_COLUMN_LABEL));
    List<SchemaPath> metadataColumns = Arrays.asList(lastModifiedTimeField, locationField, rgiField, rowGroupStartField, rowGroupLengthField);
    MetadataAggregateContext aggregateContext = MetadataAggregateContext.builder().groupByExpressions(rowGroupGroupByExpressions).interestingColumns(statisticsColumns).createNewAggregations(createNewAggregations).metadataColumns(metadataColumns).metadataLevel(MetadataType.ROW_GROUP).build();
    convertedRelNode = new MetadataAggRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode, aggregateContext);
    convertedRelNode = new MetadataHandlerRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode, handlerContext);
    return convertedRelNode;
}