Examples with RowSignature io.druid.sql.calcite.table.RowSignature used on opensource projects

Example 1 with RowSignature

use of io.druid.sql.calcite.table.RowSignature in project druid by druid-io.

the class DruidSchema method computeTable.

private DruidTable computeTable(final String dataSource) {
    final SegmentMetadataQuery segmentMetadataQuery = new SegmentMetadataQuery(new TableDataSource(dataSource), null, null, false, ImmutableMap.<String, Object>of("useCache", false, "populateCache", false), EnumSet.of(SegmentMetadataQuery.AnalysisType.INTERVAL), null, true);
    final Sequence<SegmentAnalysis> sequence = segmentMetadataQuery.run(walker, Maps.<String, Object>newHashMap());
    final List<SegmentAnalysis> results = Sequences.toList(sequence, Lists.<SegmentAnalysis>newArrayList());
    if (results.isEmpty()) {
        return null;
    }
    final Map<String, ValueType> columnTypes = Maps.newLinkedHashMap();
    // Resolve conflicts by taking the latest metadata. This aids in gradual schema evolution.
    long maxTimestamp = JodaUtils.MIN_INSTANT;
    for (SegmentAnalysis analysis : results) {
        final long timestamp;
        if (analysis.getIntervals() != null && analysis.getIntervals().size() > 0) {
            timestamp = analysis.getIntervals().get(analysis.getIntervals().size() - 1).getEndMillis();
        } else {
            timestamp = JodaUtils.MIN_INSTANT;
        }
        for (Map.Entry<String, ColumnAnalysis> entry : analysis.getColumns().entrySet()) {
            if (entry.getValue().isError()) {
                // Skip columns with analysis errors.
                continue;
            }
            if (!columnTypes.containsKey(entry.getKey()) || timestamp >= maxTimestamp) {
                ValueType valueType;
                try {
                    valueType = ValueType.valueOf(entry.getValue().getType().toUpperCase());
                } catch (IllegalArgumentException e) {
                    // Assume unrecognized types are some flavor of COMPLEX. This throws away information about exactly
                    // what kind of complex column it is, which we may want to preserve some day.
                    valueType = ValueType.COMPLEX;
                }
                columnTypes.put(entry.getKey(), valueType);
                maxTimestamp = timestamp;
            }
        }
    }
    final RowSignature.Builder rowSignature = RowSignature.builder();
    for (Map.Entry<String, ValueType> entry : columnTypes.entrySet()) {
        rowSignature.add(entry.getKey(), entry.getValue());
    }
    return new DruidTable(new TableDataSource(dataSource), rowSignature.build());
}

Example 2 with RowSignature

use of io.druid.sql.calcite.table.RowSignature in project druid by druid-io.

the class GroupByRules method applyAggregate.

/**
   * Applies a filter -> project -> aggregate chain to a druidRel. Do not call this method unless
   * {@link #canApplyAggregate(DruidRel, Filter, Project, Aggregate)} returns true.
   *
   * @return new rel, or null if the chain cannot be applied
   */
private static DruidRel applyAggregate(final DruidRel druidRel, final Filter filter0, final Project project0, final Aggregate aggregate, final DruidOperatorTable operatorTable, final boolean approximateCountDistinct) {
    Preconditions.checkState(canApplyAggregate(druidRel, filter0, project0, aggregate), "Cannot applyAggregate.");
    final RowSignature sourceRowSignature;
    final boolean isNestedQuery = druidRel.getQueryBuilder().getGrouping() != null;
    if (isNestedQuery) {
        // Nested groupBy; source row signature is the output signature of druidRel.
        sourceRowSignature = druidRel.getOutputRowSignature();
    } else {
        sourceRowSignature = druidRel.getSourceRowSignature();
    }
    // Filter that should be applied before aggregating.
    final DimFilter filter;
    if (filter0 != null) {
        filter = Expressions.toFilter(operatorTable, druidRel.getPlannerContext(), sourceRowSignature, filter0.getCondition());
        if (filter == null) {
            // Can't plan this filter.
            return null;
        }
    } else if (druidRel.getQueryBuilder().getFilter() != null && !isNestedQuery) {
        // We're going to replace the existing druidRel, so inherit its filter.
        filter = druidRel.getQueryBuilder().getFilter();
    } else {
        filter = null;
    }
    // Projection that should be applied before aggregating.
    final Project project;
    if (project0 != null) {
        project = project0;
    } else if (druidRel.getQueryBuilder().getSelectProjection() != null && !isNestedQuery) {
        // We're going to replace the existing druidRel, so inherit its projection.
        project = druidRel.getQueryBuilder().getSelectProjection().getProject();
    } else {
        project = null;
    }
    final List<DimensionSpec> dimensions = Lists.newArrayList();
    final List<Aggregation> aggregations = Lists.newArrayList();
    final List<String> rowOrder = Lists.newArrayList();
    // Translate groupSet.
    final ImmutableBitSet groupSet = aggregate.getGroupSet();
    int dimOutputNameCounter = 0;
    for (int i : groupSet) {
        if (project != null && project.getChildExps().get(i) instanceof RexLiteral) {
            // Ignore literals in GROUP BY, so a user can write e.g. "GROUP BY 'dummy'" to group everything into a single
            // row. Add dummy rowOrder entry so NULLs come out. This is not strictly correct but it works as long as
            // nobody actually expects to see the literal.
            rowOrder.add(dimOutputName(dimOutputNameCounter++));
        } else {
            final RexNode rexNode = Expressions.fromFieldAccess(sourceRowSignature, project, i);
            final RowExtraction rex = Expressions.toRowExtraction(operatorTable, druidRel.getPlannerContext(), sourceRowSignature.getRowOrder(), rexNode);
            if (rex == null) {
                return null;
            }
            final SqlTypeName sqlTypeName = rexNode.getType().getSqlTypeName();
            final ValueType outputType = Calcites.getValueTypeForSqlTypeName(sqlTypeName);
            if (outputType == null) {
                throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", sqlTypeName, rowOrder.get(i));
            }
            final DimensionSpec dimensionSpec = rex.toDimensionSpec(sourceRowSignature, dimOutputName(dimOutputNameCounter++), outputType);
            if (dimensionSpec == null) {
                return null;
            }
            dimensions.add(dimensionSpec);
            rowOrder.add(dimensionSpec.getOutputName());
        }
    }
    // Translate aggregates.
    for (int i = 0; i < aggregate.getAggCallList().size(); i++) {
        final AggregateCall aggCall = aggregate.getAggCallList().get(i);
        final Aggregation aggregation = translateAggregateCall(druidRel.getPlannerContext(), sourceRowSignature, project, aggCall, operatorTable, aggregations, i, approximateCountDistinct);
        if (aggregation == null) {
            return null;
        }
        aggregations.add(aggregation);
        rowOrder.add(aggregation.getOutputName());
    }
    if (isNestedQuery) {
        // Nested groupBy.
        return DruidNestedGroupBy.from(druidRel, filter, Grouping.create(dimensions, aggregations), aggregate.getRowType(), rowOrder);
    } else {
        // groupBy on a base dataSource.
        return druidRel.withQueryBuilder(druidRel.getQueryBuilder().withFilter(filter).withGrouping(Grouping.create(dimensions, aggregations), aggregate.getRowType(), rowOrder));
    }
}