Examples with Filter org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter used on opensource projects

Example 61 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project beam by apache.

the class BeamIOPushDownRule method onMatch.

// ~ Methods ----------------------------------------------------------------
@Override
public void onMatch(RelOptRuleCall call) {
    final BeamIOSourceRel ioSourceRel = call.rel(1);
    final BeamSqlTable beamSqlTable = ioSourceRel.getBeamSqlTable();
    if (ioSourceRel instanceof BeamPushDownIOSourceRel) {
        return;
    }
    // Nested rows are not supported at the moment
    for (RelDataTypeField field : ioSourceRel.getRowType().getFieldList()) {
        if (field.getType() instanceof RelRecordType) {
            return;
        }
    }
    final Calc calc = call.rel(0);
    final RexProgram program = calc.getProgram();
    final Pair<ImmutableList<RexNode>, ImmutableList<RexNode>> projectFilter = program.split();
    final RelDataType calcInputRowType = program.getInputRowType();
    // When predicate push-down is not supported - all filters are unsupported.
    final BeamSqlTableFilter tableFilter = beamSqlTable.constructFilter(projectFilter.right);
    if (!beamSqlTable.supportsProjects().isSupported() && tableFilter instanceof DefaultTableFilter) {
        // Either project or filter push-down must be supported by the IO.
        return;
    }
    Set<String> usedFields = new LinkedHashSet<>();
    if (!(tableFilter instanceof DefaultTableFilter) && !beamSqlTable.supportsProjects().isSupported()) {
        // When applying standalone filter push-down all fields must be project by an IO.
        // With a single exception: Calc projects all fields (in the same order) and does nothing
        // else.
        usedFields.addAll(calcInputRowType.getFieldNames());
    } else {
        // Find all input refs used by projects
        for (RexNode project : projectFilter.left) {
            findUtilizedInputRefs(calcInputRowType, project, usedFields);
        }
        // Find all input refs used by filters
        for (RexNode filter : tableFilter.getNotSupported()) {
            findUtilizedInputRefs(calcInputRowType, filter, usedFields);
        }
    }
    if (usedFields.isEmpty()) {
        // No need to do push-down for queries like this: "select UPPER('hello')".
        return;
    }
    // IO only projects fields utilized by a calc.
    if (tableFilter.getNotSupported().containsAll(projectFilter.right) && usedFields.containsAll(ioSourceRel.getRowType().getFieldNames())) {
        return;
    }
    FieldAccessDescriptor resolved = FieldAccessDescriptor.withFieldNames(usedFields);
    resolved = resolved.resolve(beamSqlTable.getSchema());
    if (canDropCalc(program, beamSqlTable.supportsProjects(), tableFilter)) {
        call.transformTo(ioSourceRel.createPushDownRel(calc.getRowType(), resolved.getFieldsAccessed().stream().map(FieldDescriptor::getFieldName).collect(Collectors.toList()), tableFilter));
        return;
    }
    // IO only projects fields utilised by a calc.
    if (tableFilter.getNotSupported().equals(projectFilter.right) && usedFields.containsAll(ioSourceRel.getRowType().getFieldNames())) {
        return;
    }
    RelNode result = constructNodesWithPushDown(resolved, call.builder(), ioSourceRel, tableFilter, calc.getRowType(), projectFilter.left);
    if (tableFilter.getNotSupported().size() <= projectFilter.right.size() || usedFields.size() < calcInputRowType.getFieldCount()) {
        // Smaller Calc programs are indisputably better, as well as IOs with less projected fields.
        // We can consider something with the same number of filters.
        call.transformTo(result);
    }
}

Example 62 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project beam by apache.

the class PubsubTableProviderIT method connect.

@SuppressWarnings("unchecked")
private CalciteConnection connect(PipelineOptions options, TableProvider... tableProviders) {
    // HACK: PipelineOptions should expose a prominent method to do this reliably
    // The actual options are in the "options" field of the converted map
    Map<String, String> argsMap = ((Map<String, Object>) MAPPER.convertValue(pipeline.getOptions(), Map.class).get("options")).entrySet().stream().filter((entry) -> {
        if (entry.getValue() instanceof List) {
            if (!((List) entry.getValue()).isEmpty()) {
                throw new IllegalArgumentException("Cannot encode list arguments");
            }
            // We can encode empty lists, just omit them.
            return false;
        }
        return true;
    }).collect(Collectors.toMap(Map.Entry::getKey, entry -> toArg(entry.getValue())));
    InMemoryMetaStore inMemoryMetaStore = new InMemoryMetaStore();
    for (TableProvider tableProvider : tableProviders) {
        inMemoryMetaStore.registerProvider(tableProvider);
    }
    JdbcConnection connection = JdbcDriver.connect(inMemoryMetaStore, options);
    connection.setPipelineOptionsMap(argsMap);
    return connection;
}

Example 63 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project beam by apache.

the class ThreeTablesSchema method testSystemNotReorderingWithoutRules.

@Test
public void testSystemNotReorderingWithoutRules() {
    TestTableProvider tableProvider = new TestTableProvider();
    createThreeTables(tableProvider);
    List<RelOptRule> ruleSet = BeamRuleSets.getRuleSets().stream().flatMap(rules -> StreamSupport.stream(rules.spliterator(), false)).filter(rule -> !(rule instanceof BeamJoinPushThroughJoinRule)).filter(rule -> !(rule instanceof BeamJoinAssociateRule)).filter(rule -> !(rule instanceof JoinCommuteRule)).collect(Collectors.toList());
    BeamSqlEnv env = BeamSqlEnv.builder(tableProvider).setPipelineOptions(PipelineOptionsFactory.create()).setRuleSets(ImmutableList.of(RuleSets.ofList(ruleSet))).build();
    // This is Join(Join(medium, large), small) which should be converted to a join that large table
    // is on the top.
    BeamRelNode parsedQuery = env.parseQuery("select * from medium_table " + " JOIN large_table on large_table.medium_key = medium_table.large_key " + " JOIN small_table on medium_table.small_key = small_table.medium_key ");
    assertTopTableInJoins(parsedQuery, "small_table");
}

Example 64 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project storm by apache.

the class StreamsFilterRule method convert.

@Override
public RelNode convert(RelNode rel) {
    final Filter filter = (Filter) rel;
    final RelNode input = filter.getInput();
    return new StreamsFilterRel(filter.getCluster(), filter.getTraitSet().replace(StreamsLogicalConvention.INSTANCE), convert(input, input.getTraitSet().replace(StreamsLogicalConvention.INSTANCE)), filter.getCondition());
}

Example 65 with Filter

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Filter in project calcite by apache.

the class CalcitePrepareImpl method analyze_.

private AnalyzeViewResult analyze_(SqlValidator validator, String sql, SqlNode sqlNode, RelRoot root, boolean fail) {
    final RexBuilder rexBuilder = root.rel.getCluster().getRexBuilder();
    RelNode rel = root.rel;
    final RelNode viewRel = rel;
    Project project;
    if (rel instanceof Project) {
        project = (Project) rel;
        rel = project.getInput();
    } else {
        project = null;
    }
    Filter filter;
    if (rel instanceof Filter) {
        filter = (Filter) rel;
        rel = filter.getInput();
    } else {
        filter = null;
    }
    TableScan scan;
    if (rel instanceof TableScan) {
        scan = (TableScan) rel;
    } else {
        scan = null;
    }
    if (scan == null) {
        if (fail) {
            throw validator.newValidationError(sqlNode, RESOURCE.modifiableViewMustBeBasedOnSingleTable());
        }
        return new AnalyzeViewResult(this, validator, sql, sqlNode, validator.getValidatedNodeType(sqlNode), root, null, null, null, null, false);
    }
    final RelOptTable targetRelTable = scan.getTable();
    final RelDataType targetRowType = targetRelTable.getRowType();
    final Table table = targetRelTable.unwrap(Table.class);
    final List<String> tablePath = targetRelTable.getQualifiedName();
    assert table != null;
    List<Integer> columnMapping;
    final Map<Integer, RexNode> projectMap = new HashMap<>();
    if (project == null) {
        columnMapping = ImmutableIntList.range(0, targetRowType.getFieldCount());
    } else {
        columnMapping = new ArrayList<>();
        for (Ord<RexNode> node : Ord.zip(project.getProjects())) {
            if (node.e instanceof RexInputRef) {
                RexInputRef rexInputRef = (RexInputRef) node.e;
                int index = rexInputRef.getIndex();
                if (projectMap.get(index) != null) {
                    if (fail) {
                        throw validator.newValidationError(sqlNode, RESOURCE.moreThanOneMappedColumn(targetRowType.getFieldList().get(index).getName(), Util.last(tablePath)));
                    }
                    return new AnalyzeViewResult(this, validator, sql, sqlNode, validator.getValidatedNodeType(sqlNode), root, null, null, null, null, false);
                }
                projectMap.put(index, rexBuilder.makeInputRef(viewRel, node.i));
                columnMapping.add(index);
            } else {
                columnMapping.add(-1);
            }
        }
    }
    final RexNode constraint;
    if (filter != null) {
        constraint = filter.getCondition();
    } else {
        constraint = rexBuilder.makeLiteral(true);
    }
    final List<RexNode> filters = new ArrayList<>();
    // If we put a constraint in projectMap above, then filters will not be empty despite
    // being a modifiable view.
    final List<RexNode> filters2 = new ArrayList<>();
    boolean retry = false;
    RelOptUtil.inferViewPredicates(projectMap, filters, constraint);
    if (fail && !filters.isEmpty()) {
        final Map<Integer, RexNode> projectMap2 = new HashMap<>();
        RelOptUtil.inferViewPredicates(projectMap2, filters2, constraint);
        if (!filters2.isEmpty()) {
            throw validator.newValidationError(sqlNode, RESOURCE.modifiableViewMustHaveOnlyEqualityPredicates());
        }
        retry = true;
    }
    // Check that all columns that are not projected have a constant value
    for (RelDataTypeField field : targetRowType.getFieldList()) {
        final int x = columnMapping.indexOf(field.getIndex());
        if (x >= 0) {
            assert Util.skip(columnMapping, x + 1).indexOf(field.getIndex()) < 0 : "column projected more than once; should have checked above";
            // target column is projected
            continue;
        }
        if (projectMap.get(field.getIndex()) != null) {
            // constant expression
            continue;
        }
        if (field.getType().isNullable()) {
            // don't need expression for nullable columns; NULL suffices
            continue;
        }
        if (fail) {
            throw validator.newValidationError(sqlNode, RESOURCE.noValueSuppliedForViewColumn(field.getName(), Util.last(tablePath)));
        }
        return new AnalyzeViewResult(this, validator, sql, sqlNode, validator.getValidatedNodeType(sqlNode), root, null, null, null, null, false);
    }
    final boolean modifiable = filters.isEmpty() || retry && filters2.isEmpty();
    return new AnalyzeViewResult(this, validator, sql, sqlNode, validator.getValidatedNodeType(sqlNode), root, modifiable ? table : null, ImmutableList.copyOf(tablePath), constraint, ImmutableIntList.copyOf(columnMapping), modifiable);
}