Search in sources :

Example 1 with SqlTypeName

use of org.apache.calcite.sql.type.SqlTypeName in project druid by druid-io.

the class SqlResource method doPost.

@POST
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public Response doPost(final SqlQuery sqlQuery) throws SQLException, IOException {
    // This is not integrated with the experimental authorization framework.
    // (Non-trivial since we don't know the dataSources up-front)
    final PlannerResult plannerResult;
    final DateTimeZone timeZone;
    try (final DruidPlanner planner = plannerFactory.createPlanner(sqlQuery.getContext())) {
        plannerResult = planner.plan(sqlQuery.getQuery());
        timeZone = planner.getPlannerContext().getTimeZone();
        // Remember which columns are time-typed, so we can emit ISO8601 instead of millis values.
        final List<RelDataTypeField> fieldList = plannerResult.rowType().getFieldList();
        final boolean[] timeColumns = new boolean[fieldList.size()];
        final boolean[] dateColumns = new boolean[fieldList.size()];
        for (int i = 0; i < fieldList.size(); i++) {
            final SqlTypeName sqlTypeName = fieldList.get(i).getType().getSqlTypeName();
            timeColumns[i] = sqlTypeName == SqlTypeName.TIMESTAMP;
            dateColumns[i] = sqlTypeName == SqlTypeName.DATE;
        }
        final Yielder<Object[]> yielder0 = Yielders.each(plannerResult.run());
        try {
            return Response.ok(new StreamingOutput() {

                @Override
                public void write(final OutputStream outputStream) throws IOException, WebApplicationException {
                    Yielder<Object[]> yielder = yielder0;
                    try (final JsonGenerator jsonGenerator = jsonMapper.getFactory().createGenerator(outputStream)) {
                        jsonGenerator.writeStartArray();
                        while (!yielder.isDone()) {
                            final Object[] row = yielder.get();
                            jsonGenerator.writeStartObject();
                            for (int i = 0; i < fieldList.size(); i++) {
                                final Object value;
                                if (timeColumns[i]) {
                                    value = ISODateTimeFormat.dateTime().print(Calcites.calciteTimestampToJoda((long) row[i], timeZone));
                                } else if (dateColumns[i]) {
                                    value = ISODateTimeFormat.dateTime().print(Calcites.calciteDateToJoda((int) row[i], timeZone));
                                } else {
                                    value = row[i];
                                }
                                jsonGenerator.writeObjectField(fieldList.get(i).getName(), value);
                            }
                            jsonGenerator.writeEndObject();
                            yielder = yielder.next(null);
                        }
                        jsonGenerator.writeEndArray();
                        jsonGenerator.flush();
                        // End with CRLF
                        outputStream.write('\r');
                        outputStream.write('\n');
                    } finally {
                        yielder.close();
                    }
                }
            }).build();
        } catch (Throwable e) {
            // make sure to close yielder if anything happened before starting to serialize the response.
            yielder0.close();
            throw Throwables.propagate(e);
        }
    } catch (Exception e) {
        log.warn(e, "Failed to handle query: %s", sqlQuery);
        final Exception exceptionToReport;
        if (e instanceof RelOptPlanner.CannotPlanException) {
            exceptionToReport = new ISE("Cannot build plan for query: %s", sqlQuery.getQuery());
        } else {
            exceptionToReport = e;
        }
        return Response.serverError().type(MediaType.APPLICATION_JSON_TYPE).entity(jsonMapper.writeValueAsBytes(QueryInterruptedException.wrapIfNeeded(exceptionToReport))).build();
    }
}
Also used : SqlTypeName(org.apache.calcite.sql.type.SqlTypeName) OutputStream(java.io.OutputStream) StreamingOutput(javax.ws.rs.core.StreamingOutput) RelOptPlanner(org.apache.calcite.plan.RelOptPlanner) DateTimeZone(org.joda.time.DateTimeZone) QueryInterruptedException(io.druid.query.QueryInterruptedException) SQLException(java.sql.SQLException) IOException(java.io.IOException) WebApplicationException(javax.ws.rs.WebApplicationException) RelDataTypeField(org.apache.calcite.rel.type.RelDataTypeField) DruidPlanner(io.druid.sql.calcite.planner.DruidPlanner) JsonGenerator(com.fasterxml.jackson.core.JsonGenerator) ISE(io.druid.java.util.common.ISE) PlannerResult(io.druid.sql.calcite.planner.PlannerResult) POST(javax.ws.rs.POST) Produces(javax.ws.rs.Produces) Consumes(javax.ws.rs.Consumes)

Example 2 with SqlTypeName

use of org.apache.calcite.sql.type.SqlTypeName in project druid by druid-io.

the class GroupByRules method translateAggregateCall.

/**
   * Translate an AggregateCall to Druid equivalents.
   *
   * @return translated aggregation, or null if translation failed.
   */
private static Aggregation translateAggregateCall(final PlannerContext plannerContext, final RowSignature sourceRowSignature, final Project project, final AggregateCall call, final DruidOperatorTable operatorTable, final List<Aggregation> existingAggregations, final int aggNumber, final boolean approximateCountDistinct) {
    final List<DimFilter> filters = Lists.newArrayList();
    final List<String> rowOrder = sourceRowSignature.getRowOrder();
    final String name = aggOutputName(aggNumber);
    final SqlKind kind = call.getAggregation().getKind();
    final SqlTypeName outputType = call.getType().getSqlTypeName();
    if (call.filterArg >= 0) {
        // AGG(xxx) FILTER(WHERE yyy)
        if (project == null) {
            // We need some kind of projection to support filtered aggregations.
            return null;
        }
        final RexNode expression = project.getChildExps().get(call.filterArg);
        final DimFilter filter = Expressions.toFilter(operatorTable, plannerContext, sourceRowSignature, expression);
        if (filter == null) {
            return null;
        }
        filters.add(filter);
    }
    if (kind == SqlKind.COUNT && call.getArgList().isEmpty()) {
        // COUNT(*)
        return Aggregation.create(new CountAggregatorFactory(name)).filter(makeFilter(filters, sourceRowSignature));
    } else if (kind == SqlKind.COUNT && call.isDistinct()) {
        // COUNT(DISTINCT x)
        return approximateCountDistinct ? APPROX_COUNT_DISTINCT.toDruidAggregation(name, sourceRowSignature, operatorTable, plannerContext, existingAggregations, project, call, makeFilter(filters, sourceRowSignature)) : null;
    } else if (kind == SqlKind.COUNT || kind == SqlKind.SUM || kind == SqlKind.SUM0 || kind == SqlKind.MIN || kind == SqlKind.MAX || kind == SqlKind.AVG) {
        // Built-in agg, not distinct, not COUNT(*)
        boolean forceCount = false;
        final FieldOrExpression input;
        final int inputField = Iterables.getOnlyElement(call.getArgList());
        final RexNode rexNode = Expressions.fromFieldAccess(sourceRowSignature, project, inputField);
        final FieldOrExpression foe = FieldOrExpression.fromRexNode(operatorTable, plannerContext, rowOrder, rexNode);
        if (foe != null) {
            input = foe;
        } else if (rexNode.getKind() == SqlKind.CASE && ((RexCall) rexNode).getOperands().size() == 3) {
            // Possibly a CASE-style filtered aggregation. Styles supported:
            // A: SUM(CASE WHEN x = 'foo' THEN cnt END) => operands (x = 'foo', cnt, null)
            // B: SUM(CASE WHEN x = 'foo' THEN 1 ELSE 0 END) => operands (x = 'foo', 1, 0)
            // C: COUNT(CASE WHEN x = 'foo' THEN 'dummy' END) => operands (x = 'foo', 'dummy', null)
            // If the null and non-null args are switched, "flip" is set, which negates the filter.
            final RexCall caseCall = (RexCall) rexNode;
            final boolean flip = RexLiteral.isNullLiteral(caseCall.getOperands().get(1)) && !RexLiteral.isNullLiteral(caseCall.getOperands().get(2));
            final RexNode arg1 = caseCall.getOperands().get(flip ? 2 : 1);
            final RexNode arg2 = caseCall.getOperands().get(flip ? 1 : 2);
            // Operand 1: Filter
            final DimFilter filter = Expressions.toFilter(operatorTable, plannerContext, sourceRowSignature, caseCall.getOperands().get(0));
            if (filter == null) {
                return null;
            } else {
                filters.add(flip ? new NotDimFilter(filter) : filter);
            }
            if (call.getAggregation().getKind() == SqlKind.COUNT && arg1 instanceof RexLiteral && !RexLiteral.isNullLiteral(arg1) && RexLiteral.isNullLiteral(arg2)) {
                // Case C
                forceCount = true;
                input = null;
            } else if (call.getAggregation().getKind() == SqlKind.SUM && arg1 instanceof RexLiteral && ((Number) RexLiteral.value(arg1)).intValue() == 1 && arg2 instanceof RexLiteral && ((Number) RexLiteral.value(arg2)).intValue() == 0) {
                // Case B
                forceCount = true;
                input = null;
            } else if (RexLiteral.isNullLiteral(arg2)) {
                // Maybe case A
                input = FieldOrExpression.fromRexNode(operatorTable, plannerContext, rowOrder, arg1);
                if (input == null) {
                    return null;
                }
            } else {
                // Can't translate CASE into a filter.
                return null;
            }
        } else {
            // Can't translate operand.
            return null;
        }
        if (!forceCount) {
            Preconditions.checkNotNull(input, "WTF?! input was null for non-COUNT aggregation");
        }
        if (forceCount || kind == SqlKind.COUNT) {
            // COUNT(x)
            return Aggregation.create(new CountAggregatorFactory(name)).filter(makeFilter(filters, sourceRowSignature));
        } else {
            // Built-in aggregator that is not COUNT.
            final Aggregation retVal;
            final String fieldName = input.getFieldName();
            final String expression = input.getExpression();
            final boolean isLong = SqlTypeName.INT_TYPES.contains(outputType) || SqlTypeName.TIMESTAMP == outputType || SqlTypeName.DATE == outputType;
            if (kind == SqlKind.SUM || kind == SqlKind.SUM0) {
                retVal = isLong ? Aggregation.create(new LongSumAggregatorFactory(name, fieldName, expression)) : Aggregation.create(new DoubleSumAggregatorFactory(name, fieldName, expression));
            } else if (kind == SqlKind.MIN) {
                retVal = isLong ? Aggregation.create(new LongMinAggregatorFactory(name, fieldName, expression)) : Aggregation.create(new DoubleMinAggregatorFactory(name, fieldName, expression));
            } else if (kind == SqlKind.MAX) {
                retVal = isLong ? Aggregation.create(new LongMaxAggregatorFactory(name, fieldName, expression)) : Aggregation.create(new DoubleMaxAggregatorFactory(name, fieldName, expression));
            } else if (kind == SqlKind.AVG) {
                final String sumName = aggInternalName(aggNumber, "sum");
                final String countName = aggInternalName(aggNumber, "count");
                final AggregatorFactory sum = isLong ? new LongSumAggregatorFactory(sumName, fieldName, expression) : new DoubleSumAggregatorFactory(sumName, fieldName, expression);
                final AggregatorFactory count = new CountAggregatorFactory(countName);
                retVal = Aggregation.create(ImmutableList.of(sum, count), new ArithmeticPostAggregator(name, "quotient", ImmutableList.<PostAggregator>of(new FieldAccessPostAggregator(null, sumName), new FieldAccessPostAggregator(null, countName))));
            } else {
                // Not reached.
                throw new ISE("WTF?! Kind[%s] got into the built-in aggregator path somehow?!", kind);
            }
            return retVal.filter(makeFilter(filters, sourceRowSignature));
        }
    } else {
        // Not a built-in aggregator, check operator table.
        final SqlAggregator sqlAggregator = operatorTable.lookupAggregator(call.getAggregation().getName());
        return sqlAggregator != null ? sqlAggregator.toDruidAggregation(name, sourceRowSignature, operatorTable, plannerContext, existingAggregations, project, call, makeFilter(filters, sourceRowSignature)) : null;
    }
}
Also used : RexLiteral(org.apache.calcite.rex.RexLiteral) ArithmeticPostAggregator(io.druid.query.aggregation.post.ArithmeticPostAggregator) DoubleMaxAggregatorFactory(io.druid.query.aggregation.DoubleMaxAggregatorFactory) SqlTypeName(org.apache.calcite.sql.type.SqlTypeName) LongSumAggregatorFactory(io.druid.query.aggregation.LongSumAggregatorFactory) DoubleMinAggregatorFactory(io.druid.query.aggregation.DoubleMinAggregatorFactory) LongMinAggregatorFactory(io.druid.query.aggregation.LongMinAggregatorFactory) RexCall(org.apache.calcite.rex.RexCall) Aggregation(io.druid.sql.calcite.aggregation.Aggregation) ISE(io.druid.java.util.common.ISE) LongMaxAggregatorFactory(io.druid.query.aggregation.LongMaxAggregatorFactory) NotDimFilter(io.druid.query.filter.NotDimFilter) FieldAccessPostAggregator(io.druid.query.aggregation.post.FieldAccessPostAggregator) DoubleSumAggregatorFactory(io.druid.query.aggregation.DoubleSumAggregatorFactory) PostAggregator(io.druid.query.aggregation.PostAggregator) FieldAccessPostAggregator(io.druid.query.aggregation.post.FieldAccessPostAggregator) ArithmeticPostAggregator(io.druid.query.aggregation.post.ArithmeticPostAggregator) SqlKind(org.apache.calcite.sql.SqlKind) CountAggregatorFactory(io.druid.query.aggregation.CountAggregatorFactory) DoubleMaxAggregatorFactory(io.druid.query.aggregation.DoubleMaxAggregatorFactory) LongMaxAggregatorFactory(io.druid.query.aggregation.LongMaxAggregatorFactory) DoubleSumAggregatorFactory(io.druid.query.aggregation.DoubleSumAggregatorFactory) AggregatorFactory(io.druid.query.aggregation.AggregatorFactory) LongMinAggregatorFactory(io.druid.query.aggregation.LongMinAggregatorFactory) PostAggregatorFactory(io.druid.sql.calcite.aggregation.PostAggregatorFactory) DoubleMinAggregatorFactory(io.druid.query.aggregation.DoubleMinAggregatorFactory) LongSumAggregatorFactory(io.druid.query.aggregation.LongSumAggregatorFactory) CountAggregatorFactory(io.druid.query.aggregation.CountAggregatorFactory) SqlAggregator(io.druid.sql.calcite.aggregation.SqlAggregator) ApproxCountDistinctSqlAggregator(io.druid.sql.calcite.aggregation.ApproxCountDistinctSqlAggregator) DimFilter(io.druid.query.filter.DimFilter) NotDimFilter(io.druid.query.filter.NotDimFilter) AndDimFilter(io.druid.query.filter.AndDimFilter) RexNode(org.apache.calcite.rex.RexNode)

Example 3 with SqlTypeName

use of org.apache.calcite.sql.type.SqlTypeName in project druid by druid-io.

the class ApproxCountDistinctSqlAggregator method toDruidAggregation.

@Override
public Aggregation toDruidAggregation(final String name, final RowSignature rowSignature, final DruidOperatorTable operatorTable, final PlannerContext plannerContext, final List<Aggregation> existingAggregations, final Project project, final AggregateCall aggregateCall, final DimFilter filter) {
    final RexNode rexNode = Expressions.fromFieldAccess(rowSignature, project, Iterables.getOnlyElement(aggregateCall.getArgList()));
    final RowExtraction rex = Expressions.toRowExtraction(operatorTable, plannerContext, rowSignature.getRowOrder(), rexNode);
    if (rex == null) {
        return null;
    }
    final AggregatorFactory aggregatorFactory;
    if (rowSignature.getColumnType(rex.getColumn()) == ValueType.COMPLEX) {
        aggregatorFactory = new HyperUniquesAggregatorFactory(name, rex.getColumn());
    } else {
        final SqlTypeName sqlTypeName = rexNode.getType().getSqlTypeName();
        final ValueType outputType = Calcites.getValueTypeForSqlTypeName(sqlTypeName);
        if (outputType == null) {
            throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", sqlTypeName, name);
        }
        final DimensionSpec dimensionSpec = rex.toDimensionSpec(rowSignature, null, ValueType.STRING);
        if (dimensionSpec == null) {
            return null;
        }
        aggregatorFactory = new CardinalityAggregatorFactory(name, ImmutableList.of(dimensionSpec), false);
    }
    return Aggregation.createFinalizable(ImmutableList.<AggregatorFactory>of(aggregatorFactory), null, new PostAggregatorFactory() {

        @Override
        public PostAggregator factorize(String outputName) {
            return new HyperUniqueFinalizingPostAggregator(outputName, name);
        }
    }).filter(filter);
}
Also used : DimensionSpec(io.druid.query.dimension.DimensionSpec) SqlTypeName(org.apache.calcite.sql.type.SqlTypeName) ValueType(io.druid.segment.column.ValueType) HyperUniquesAggregatorFactory(io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory) RowExtraction(io.druid.sql.calcite.expression.RowExtraction) ISE(io.druid.java.util.common.ISE) HyperUniqueFinalizingPostAggregator(io.druid.query.aggregation.hyperloglog.HyperUniqueFinalizingPostAggregator) AggregatorFactory(io.druid.query.aggregation.AggregatorFactory) HyperUniquesAggregatorFactory(io.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory) CardinalityAggregatorFactory(io.druid.query.aggregation.cardinality.CardinalityAggregatorFactory) CardinalityAggregatorFactory(io.druid.query.aggregation.cardinality.CardinalityAggregatorFactory) RexNode(org.apache.calcite.rex.RexNode)

Example 4 with SqlTypeName

use of org.apache.calcite.sql.type.SqlTypeName in project druid by druid-io.

the class Expressions method toMathExpression.

/**
   * Translate a row-expression to a Druid math expression. One day, when possible, this could be folded into
   * {@link #toRowExtraction(DruidOperatorTable, PlannerContext, List, RexNode)}.
   *
   * @param rowOrder   order of fields in the Druid rows to be extracted from
   * @param expression expression meant to be applied on top of the rows
   *
   * @return expression referring to fields in rowOrder, or null if not possible
   */
public static String toMathExpression(final List<String> rowOrder, final RexNode expression) {
    final SqlKind kind = expression.getKind();
    final SqlTypeName sqlTypeName = expression.getType().getSqlTypeName();
    if (kind == SqlKind.INPUT_REF) {
        // Translate field references.
        final RexInputRef ref = (RexInputRef) expression;
        final String columnName = rowOrder.get(ref.getIndex());
        if (columnName == null) {
            throw new ISE("WTF?! Expression referred to nonexistent index[%d]", ref.getIndex());
        }
        return String.format("\"%s\"", escape(columnName));
    } else if (kind == SqlKind.CAST || kind == SqlKind.REINTERPRET) {
        // Translate casts.
        final RexNode operand = ((RexCall) expression).getOperands().get(0);
        final String operandExpression = toMathExpression(rowOrder, operand);
        if (operandExpression == null) {
            return null;
        }
        final ExprType fromType = MATH_TYPES.get(operand.getType().getSqlTypeName());
        final ExprType toType = MATH_TYPES.get(sqlTypeName);
        if (fromType != toType) {
            return String.format("CAST(%s, '%s')", operandExpression, toType.toString());
        } else {
            return operandExpression;
        }
    } else if (kind == SqlKind.TIMES || kind == SqlKind.DIVIDE || kind == SqlKind.PLUS || kind == SqlKind.MINUS) {
        // Translate simple arithmetic.
        final List<RexNode> operands = ((RexCall) expression).getOperands();
        final String lhsExpression = toMathExpression(rowOrder, operands.get(0));
        final String rhsExpression = toMathExpression(rowOrder, operands.get(1));
        if (lhsExpression == null || rhsExpression == null) {
            return null;
        }
        final String op = ImmutableMap.of(SqlKind.TIMES, "*", SqlKind.DIVIDE, "/", SqlKind.PLUS, "+", SqlKind.MINUS, "-").get(kind);
        return String.format("(%s %s %s)", lhsExpression, op, rhsExpression);
    } else if (kind == SqlKind.OTHER_FUNCTION) {
        final String calciteFunction = ((RexCall) expression).getOperator().getName();
        final String druidFunction = MATH_FUNCTIONS.get(calciteFunction);
        final List<String> functionArgs = Lists.newArrayList();
        for (final RexNode operand : ((RexCall) expression).getOperands()) {
            final String operandExpression = toMathExpression(rowOrder, operand);
            if (operandExpression == null) {
                return null;
            }
            functionArgs.add(operandExpression);
        }
        if ("MOD".equals(calciteFunction)) {
            // Special handling for MOD, which is a function in Calcite but a binary operator in Druid.
            Preconditions.checkState(functionArgs.size() == 2, "WTF?! Expected 2 args for MOD.");
            return String.format("(%s %s %s)", functionArgs.get(0), "%", functionArgs.get(1));
        }
        if (druidFunction == null) {
            return null;
        }
        return String.format("%s(%s)", druidFunction, Joiner.on(", ").join(functionArgs));
    } else if (kind == SqlKind.LITERAL) {
        // Translate literal.
        if (SqlTypeName.NUMERIC_TYPES.contains(sqlTypeName)) {
            // Include literal numbers as-is.
            return String.valueOf(RexLiteral.value(expression));
        } else if (SqlTypeName.STRING_TYPES.contains(sqlTypeName)) {
            // Quote literal strings.
            return "\'" + escape(RexLiteral.stringValue(expression)) + "\'";
        } else {
            // Can't translate other literals.
            return null;
        }
    } else {
        // Can't translate other kinds of expressions.
        return null;
    }
}
Also used : RexCall(org.apache.calcite.rex.RexCall) SqlTypeName(org.apache.calcite.sql.type.SqlTypeName) ExprType(io.druid.math.expr.ExprType) RexInputRef(org.apache.calcite.rex.RexInputRef) ISE(io.druid.java.util.common.ISE) List(java.util.List) SqlKind(org.apache.calcite.sql.SqlKind) RexNode(org.apache.calcite.rex.RexNode)

Example 5 with SqlTypeName

use of org.apache.calcite.sql.type.SqlTypeName in project druid by druid-io.

the class Expressions method toMillisLiteral.

/**
   * Translates "literal" (a TIMESTAMP or DATE literal) to milliseconds since the epoch using the provided
   * session time zone.
   *
   * @param literal  TIMESTAMP or DATE literal
   * @param timeZone session time zone
   *
   * @return milliseconds time
   */
public static long toMillisLiteral(final RexNode literal, final DateTimeZone timeZone) {
    final SqlTypeName typeName = literal.getType().getSqlTypeName();
    if (literal.getKind() != SqlKind.LITERAL || (typeName != SqlTypeName.TIMESTAMP && typeName != SqlTypeName.DATE)) {
        throw new IAE("Expected TIMESTAMP or DATE literal but got[%s:%s]", literal.getKind(), typeName);
    }
    final Calendar calendar = (Calendar) RexLiteral.value(literal);
    return Calcites.calciteTimestampToJoda(calendar.getTimeInMillis(), timeZone).getMillis();
}
Also used : SqlTypeName(org.apache.calcite.sql.type.SqlTypeName) Calendar(java.util.Calendar) IAE(io.druid.java.util.common.IAE)

Aggregations

SqlTypeName (org.apache.calcite.sql.type.SqlTypeName)116 Test (org.junit.Test)38 RelDataType (org.apache.calcite.rel.type.RelDataType)28 RexNode (org.apache.calcite.rex.RexNode)18 BigDecimal (java.math.BigDecimal)13 ArrayList (java.util.ArrayList)12 List (java.util.List)11 RelDataTypeFactory (org.apache.calcite.rel.type.RelDataTypeFactory)8 RelDataTypeField (org.apache.calcite.rel.type.RelDataTypeField)8 ImmutableList (com.google.common.collect.ImmutableList)7 Map (java.util.Map)7 DateString (org.apache.calcite.util.DateString)7 TimeString (org.apache.calcite.util.TimeString)7 TimestampString (org.apache.calcite.util.TimestampString)7 ISE (io.druid.java.util.common.ISE)6 SqlKind (org.apache.calcite.sql.SqlKind)6 NlsString (org.apache.calcite.util.NlsString)6 Calendar (java.util.Calendar)5 Nullable (javax.annotation.Nullable)5 RexBuilder (org.apache.calcite.rex.RexBuilder)5