Search in sources :

Example 1 with COMPARISON

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.COMPARISON in project beam by apache.

the class MongoDbTable method translateRexNodeToBson.

/**
 * Recursively translates a single RexNode to MongoDB Bson filter. Supports simple comparison
 * operations, negation, and nested conjunction/disjunction. Boolean fields are translated as an
 * `$eq` operation with a boolean `true`.
 *
 * @param node {@code RexNode} to translate.
 * @return {@code Bson} filter.
 */
private Bson translateRexNodeToBson(RexNode node) {
    final IntFunction<String> fieldIdToName = i -> getSchema().getField(i).getName();
    // Supported operations are described in MongoDbFilter#isSupported
    if (node instanceof RexCall) {
        RexCall compositeNode = (RexCall) node;
        List<RexLiteral> literals = new ArrayList<>();
        List<RexInputRef> inputRefs = new ArrayList<>();
        for (RexNode operand : compositeNode.getOperands()) {
            if (operand instanceof RexLiteral) {
                literals.add((RexLiteral) operand);
            } else if (operand instanceof RexInputRef) {
                inputRefs.add((RexInputRef) operand);
            }
        }
        // Operation is a comparison, since one of the operands in a field reference.
        if (inputRefs.size() == 1) {
            RexInputRef inputRef = inputRefs.get(0);
            String inputFieldName = fieldIdToName.apply(inputRef.getIndex());
            if (literals.size() > 0) {
                // Convert literal value to the same Java type as the field we are comparing to.
                Object literal = convertToExpectedType(inputRef, literals.get(0));
                switch(node.getKind()) {
                    case IN:
                        return Filters.in(inputFieldName, convertToExpectedType(inputRef, literals));
                    case EQUALS:
                        return Filters.eq(inputFieldName, literal);
                    case NOT_EQUALS:
                        return Filters.not(Filters.eq(inputFieldName, literal));
                    case LESS_THAN:
                        return Filters.lt(inputFieldName, literal);
                    case GREATER_THAN:
                        return Filters.gt(inputFieldName, literal);
                    case GREATER_THAN_OR_EQUAL:
                        return Filters.gte(inputFieldName, literal);
                    case LESS_THAN_OR_EQUAL:
                        return Filters.lte(inputFieldName, literal);
                    default:
                        // Encountered an unexpected node kind, RuntimeException below.
                        break;
                }
            } else if (node.getKind().equals(SqlKind.NOT)) {
                // Ex: `where not boolean_field`
                return Filters.not(translateRexNodeToBson(inputRef));
            } else {
                throw new RuntimeException("Cannot create a filter for an unsupported node: " + node.toString());
            }
        } else {
            // Operation is a conjunction/disjunction.
            switch(node.getKind()) {
                case AND:
                    // Recursively construct filter for each operand of conjunction.
                    return Filters.and(compositeNode.getOperands().stream().map(this::translateRexNodeToBson).collect(Collectors.toList()));
                case OR:
                    // Recursively construct filter for each operand of disjunction.
                    return Filters.or(compositeNode.getOperands().stream().map(this::translateRexNodeToBson).collect(Collectors.toList()));
                default:
                    // Encountered an unexpected node kind, RuntimeException below.
                    break;
            }
        }
        throw new RuntimeException("Encountered an unexpected node kind: " + node.getKind().toString());
    } else if (node instanceof RexInputRef && node.getType().getSqlTypeName().equals(SqlTypeName.BOOLEAN)) {
        // Boolean field, must be true. Ex: `select * from table where bool_field`
        return Filters.eq(fieldIdToName.apply(((RexInputRef) node).getIndex()), true);
    }
    throw new RuntimeException("Was expecting a RexCall or a boolean RexInputRef, but received: " + node.getClass().getSimpleName());
}
Also used : Document(org.bson.Document) PBegin(org.apache.beam.sdk.values.PBegin) LoggerFactory(org.slf4j.LoggerFactory) MongoDbIO(org.apache.beam.sdk.io.mongodb.MongoDbIO) IsBounded(org.apache.beam.sdk.values.PCollection.IsBounded) JsonToRow(org.apache.beam.sdk.transforms.JsonToRow) SimpleFunction(org.apache.beam.sdk.transforms.SimpleFunction) ProjectSupport(org.apache.beam.sdk.extensions.sql.meta.ProjectSupport) Table(org.apache.beam.sdk.extensions.sql.meta.Table) Matcher(java.util.regex.Matcher) AND(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND) RexLiteral(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral) MapElements(org.apache.beam.sdk.transforms.MapElements) SchemaBaseBeamTable(org.apache.beam.sdk.extensions.sql.meta.SchemaBaseBeamTable) FieldType(org.apache.beam.sdk.schemas.Schema.FieldType) Collectors(java.util.stream.Collectors) FindQuery(org.apache.beam.sdk.io.mongodb.FindQuery) Serializable(java.io.Serializable) InvalidTableException(org.apache.beam.sdk.extensions.sql.meta.provider.InvalidTableException) VisibleForTesting(org.apache.beam.vendor.calcite.v1_28_0.com.google.common.annotations.VisibleForTesting) COMPARISON(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.COMPARISON) POutput(org.apache.beam.sdk.values.POutput) List(java.util.List) ParDo(org.apache.beam.sdk.transforms.ParDo) BeamTableStatistics(org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics) ImmutableList(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList) Pattern(java.util.regex.Pattern) Experimental(org.apache.beam.sdk.annotations.Experimental) RexNode(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode) JsonWriterSettings(org.bson.json.JsonWriterSettings) SqlKind(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind) SelectHelpers(org.apache.beam.sdk.schemas.utils.SelectHelpers) ArrayList(java.util.ArrayList) PTransform(org.apache.beam.sdk.transforms.PTransform) Filters(com.mongodb.client.model.Filters) DefaultTableFilter(org.apache.beam.sdk.extensions.sql.meta.DefaultTableFilter) Bson(org.bson.conversions.Bson) ToJson(org.apache.beam.sdk.transforms.ToJson) JsonMode(org.bson.json.JsonMode) FieldAccessDescriptor(org.apache.beam.sdk.schemas.FieldAccessDescriptor) Row(org.apache.beam.sdk.values.Row) PipelineOptions(org.apache.beam.sdk.options.PipelineOptions) IntFunction(java.util.function.IntFunction) FieldTypeDescriptors(org.apache.beam.sdk.schemas.FieldTypeDescriptors) DoFn(org.apache.beam.sdk.transforms.DoFn) Logger(org.slf4j.Logger) BeamSqlTableFilter(org.apache.beam.sdk.extensions.sql.meta.BeamSqlTableFilter) PCollection(org.apache.beam.sdk.values.PCollection) Schema(org.apache.beam.sdk.schemas.Schema) RexCall(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexCall) SqlTypeName(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.type.SqlTypeName) RexInputRef(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef) OR(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.OR) RexCall(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexCall) RexLiteral(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral) ArrayList(java.util.ArrayList) RexInputRef(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef) RexNode(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode)

Example 2 with COMPARISON

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.COMPARISON in project beam by apache.

the class BigQueryFilter method isSupported.

/**
 * Check whether a {@code RexNode} is supported. As of right now BigQuery supports: 1. Complex
 * predicates (both conjunction and disjunction). 2. Comparison between a column and a literal.
 *
 * <p>TODO: Check if comparison between two columns is supported. Also over a boolean field.
 *
 * @param node A node to check for predicate push-down support.
 * @return A pair containing a boolean whether an expression is supported and the number of input
 *     references used by the expression.
 */
private Pair<Boolean, Integer> isSupported(RexNode node) {
    int numberOfInputRefs = 0;
    boolean isSupported = true;
    if (node instanceof RexCall) {
        RexCall compositeNode = (RexCall) node;
        // CAST, TRIM? and REVERSE? should be supported as well.
        if (!node.getKind().belongsTo(SUPPORTED_OPS)) {
            isSupported = false;
        } else {
            for (RexNode operand : compositeNode.getOperands()) {
                // All operands must be supported for a parent node to be supported.
                Pair<Boolean, Integer> childSupported = isSupported(operand);
                // (OR).
                if (!node.getKind().belongsTo(ImmutableSet.of(AND, OR))) {
                    numberOfInputRefs += childSupported.getRight();
                }
                // Predicate functions, where more than one field is involved are unsupported.
                isSupported = numberOfInputRefs < 2 && childSupported.getLeft();
            }
        }
    } else if (node instanceof RexInputRef) {
        numberOfInputRefs = 1;
    } else if (node instanceof RexLiteral) {
    // RexLiterals are expected, but no action is needed.
    } else {
        throw new UnsupportedOperationException("Encountered an unexpected node type: " + node.getClass().getSimpleName());
    }
    return Pair.of(isSupported, numberOfInputRefs);
}
Also used : RexCall(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexCall) RexLiteral(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral) RexInputRef(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef) RexNode(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode)

Aggregations

RexCall (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexCall)2 RexInputRef (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexInputRef)2 RexLiteral (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexLiteral)2 RexNode (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rex.RexNode)2 Filters (com.mongodb.client.model.Filters)1 Serializable (java.io.Serializable)1 ArrayList (java.util.ArrayList)1 List (java.util.List)1 IntFunction (java.util.function.IntFunction)1 Matcher (java.util.regex.Matcher)1 Pattern (java.util.regex.Pattern)1 Collectors (java.util.stream.Collectors)1 Experimental (org.apache.beam.sdk.annotations.Experimental)1 BeamTableStatistics (org.apache.beam.sdk.extensions.sql.impl.BeamTableStatistics)1 BeamSqlTableFilter (org.apache.beam.sdk.extensions.sql.meta.BeamSqlTableFilter)1 DefaultTableFilter (org.apache.beam.sdk.extensions.sql.meta.DefaultTableFilter)1 ProjectSupport (org.apache.beam.sdk.extensions.sql.meta.ProjectSupport)1 SchemaBaseBeamTable (org.apache.beam.sdk.extensions.sql.meta.SchemaBaseBeamTable)1 Table (org.apache.beam.sdk.extensions.sql.meta.Table)1 InvalidTableException (org.apache.beam.sdk.extensions.sql.meta.provider.InvalidTableException)1