Search in sources :

Example 6 with LogicalJoin

use of org.apache.calcite.rel.logical.LogicalJoin in project calcite by apache.

the class SqlToRelConverter method substituteSubQuery.

private void substituteSubQuery(Blackboard bb, SubQuery subQuery) {
    final RexNode expr = subQuery.expr;
    if (expr != null) {
        // Already done.
        return;
    }
    final SqlBasicCall call;
    final RelNode rel;
    final SqlNode query;
    final RelOptUtil.Exists converted;
    switch(subQuery.node.getKind()) {
        case CURSOR:
            convertCursor(bb, subQuery);
            return;
        case MULTISET_QUERY_CONSTRUCTOR:
        case MULTISET_VALUE_CONSTRUCTOR:
        case ARRAY_QUERY_CONSTRUCTOR:
            rel = convertMultisets(ImmutableList.of(subQuery.node), bb);
            subQuery.expr = bb.register(rel, JoinRelType.INNER);
            return;
        case IN:
        case NOT_IN:
        case SOME:
        case ALL:
            call = (SqlBasicCall) subQuery.node;
            query = call.operand(1);
            if (!config.isExpand() && !(query instanceof SqlNodeList)) {
                return;
            }
            final SqlNode leftKeyNode = call.operand(0);
            final List<RexNode> leftKeys;
            switch(leftKeyNode.getKind()) {
                case ROW:
                    leftKeys = Lists.newArrayList();
                    for (SqlNode sqlExpr : ((SqlBasicCall) leftKeyNode).getOperandList()) {
                        leftKeys.add(bb.convertExpression(sqlExpr));
                    }
                    break;
                default:
                    leftKeys = ImmutableList.of(bb.convertExpression(leftKeyNode));
            }
            if (query instanceof SqlNodeList) {
                SqlNodeList valueList = (SqlNodeList) query;
                if (!containsNullLiteral(valueList) && valueList.size() < config.getInSubQueryThreshold()) {
                    // We're under the threshold, so convert to OR.
                    subQuery.expr = convertInToOr(bb, leftKeys, valueList, (SqlInOperator) call.getOperator());
                    return;
                }
            // Otherwise, let convertExists translate
            // values list into an inline table for the
            // reference to Q below.
            }
            // Project out the search columns from the left side
            // Q1:
            // "select from emp where emp.deptno in (select col1 from T)"
            // 
            // is converted to
            // 
            // "select from
            // emp inner join (select distinct col1 from T)) q
            // on emp.deptno = q.col1
            // 
            // Q2:
            // "select from emp where emp.deptno not in (Q)"
            // 
            // is converted to
            // 
            // "select from
            // emp left outer join (select distinct col1, TRUE from T) q
            // on emp.deptno = q.col1
            // where emp.deptno <> null
            // and q.indicator <> TRUE"
            // 
            final RelDataType targetRowType = SqlTypeUtil.promoteToRowType(typeFactory, validator.getValidatedNodeType(leftKeyNode), null);
            final boolean notIn = call.getOperator().kind == SqlKind.NOT_IN;
            converted = convertExists(query, RelOptUtil.SubQueryType.IN, subQuery.logic, notIn, targetRowType);
            if (converted.indicator) {
                // Generate
                // emp CROSS JOIN (SELECT COUNT(*) AS c,
                // COUNT(deptno) AS ck FROM dept)
                final RelDataType longType = typeFactory.createSqlType(SqlTypeName.BIGINT);
                // fragile
                final RelNode seek = converted.r.getInput(0);
                final int keyCount = leftKeys.size();
                final List<Integer> args = ImmutableIntList.range(0, keyCount);
                LogicalAggregate aggregate = LogicalAggregate.create(seek, ImmutableBitSet.of(), null, ImmutableList.of(AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, ImmutableList.<Integer>of(), -1, longType, null), AggregateCall.create(SqlStdOperatorTable.COUNT, false, false, args, -1, longType, null)));
                LogicalJoin join = LogicalJoin.create(bb.root, aggregate, rexBuilder.makeLiteral(true), ImmutableSet.<CorrelationId>of(), JoinRelType.INNER);
                bb.setRoot(join, false);
            }
            final RexNode rex = bb.register(converted.r, converted.outerJoin ? JoinRelType.LEFT : JoinRelType.INNER, leftKeys);
            RelOptUtil.Logic logic = subQuery.logic;
            switch(logic) {
                case TRUE_FALSE_UNKNOWN:
                case UNKNOWN_AS_TRUE:
                    if (!converted.indicator) {
                        logic = RelOptUtil.Logic.TRUE_FALSE;
                    }
            }
            subQuery.expr = translateIn(logic, bb.root, rex);
            if (notIn) {
                subQuery.expr = rexBuilder.makeCall(SqlStdOperatorTable.NOT, subQuery.expr);
            }
            return;
        case EXISTS:
            // "select from emp where exists (select a from T)"
            // 
            // is converted to the following if the sub-query is correlated:
            // 
            // "select from emp left outer join (select AGG_TRUE() as indicator
            // from T group by corr_var) q where q.indicator is true"
            // 
            // If there is no correlation, the expression is replaced with a
            // boolean indicating whether the sub-query returned 0 or >= 1 row.
            call = (SqlBasicCall) subQuery.node;
            query = call.operand(0);
            if (!config.isExpand()) {
                return;
            }
            converted = convertExists(query, RelOptUtil.SubQueryType.EXISTS, subQuery.logic, true, null);
            assert !converted.indicator;
            if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, true)) {
                return;
            }
            subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
            return;
        case SCALAR_QUERY:
            // to a constant expression.
            if (!config.isExpand()) {
                return;
            }
            call = (SqlBasicCall) subQuery.node;
            query = call.operand(0);
            converted = convertExists(query, RelOptUtil.SubQueryType.SCALAR, subQuery.logic, true, null);
            assert !converted.indicator;
            if (convertNonCorrelatedSubQuery(subQuery, bb, converted.r, false)) {
                return;
            }
            rel = convertToSingleValueSubq(query, converted.r);
            subQuery.expr = bb.register(rel, JoinRelType.LEFT);
            return;
        case SELECT:
            // This is used when converting multiset queries:
            // 
            // select * from unnest(select multiset[deptno] from emps);
            // 
            converted = convertExists(subQuery.node, RelOptUtil.SubQueryType.SCALAR, subQuery.logic, true, null);
            assert !converted.indicator;
            subQuery.expr = bb.register(converted.r, JoinRelType.LEFT);
            return;
        default:
            throw new AssertionError("unexpected kind of sub-query: " + subQuery.node);
    }
}
Also used : RelOptUtil(org.apache.calcite.plan.RelOptUtil) SqlInOperator(org.apache.calcite.sql.fun.SqlInOperator) RelDataType(org.apache.calcite.rel.type.RelDataType) LogicalAggregate(org.apache.calcite.rel.logical.LogicalAggregate) SqlBasicCall(org.apache.calcite.sql.SqlBasicCall) RelNode(org.apache.calcite.rel.RelNode) LogicalJoin(org.apache.calcite.rel.logical.LogicalJoin) SqlNodeList(org.apache.calcite.sql.SqlNodeList) RexNode(org.apache.calcite.rex.RexNode) SqlNode(org.apache.calcite.sql.SqlNode)

Example 7 with LogicalJoin

use of org.apache.calcite.rel.logical.LogicalJoin in project samza by apache.

the class TestJoinTranslator method testTranslateStreamToTableJoin.

private void testTranslateStreamToTableJoin(boolean isRemoteTable) throws IOException, ClassNotFoundException {
    // setup mock values to the constructor of JoinTranslator
    final String logicalOpId = "sql0_join3";
    final int queryId = 0;
    LogicalJoin mockJoin = PowerMockito.mock(LogicalJoin.class);
    TranslatorContext mockTranslatorContext = mock(TranslatorContext.class);
    RelNode mockLeftInput = PowerMockito.mock(LogicalTableScan.class);
    RelNode mockRightInput = mock(RelNode.class);
    List<RelNode> inputs = new ArrayList<>();
    inputs.add(mockLeftInput);
    inputs.add(mockRightInput);
    RelOptTable mockLeftTable = mock(RelOptTable.class);
    when(mockLeftInput.getTable()).thenReturn(mockLeftTable);
    List<String> qualifiedTableName = Arrays.asList("test", "LeftTable");
    when(mockLeftTable.getQualifiedName()).thenReturn(qualifiedTableName);
    when(mockLeftInput.getId()).thenReturn(1);
    when(mockRightInput.getId()).thenReturn(2);
    when(mockJoin.getId()).thenReturn(3);
    when(mockJoin.getInputs()).thenReturn(inputs);
    when(mockJoin.getLeft()).thenReturn(mockLeftInput);
    when(mockJoin.getRight()).thenReturn(mockRightInput);
    RexCall mockJoinCondition = mock(RexCall.class);
    when(mockJoinCondition.isAlwaysTrue()).thenReturn(false);
    when(mockJoinCondition.getKind()).thenReturn(SqlKind.EQUALS);
    when(mockJoin.getCondition()).thenReturn(mockJoinCondition);
    RexInputRef mockLeftConditionInput = mock(RexInputRef.class);
    RexInputRef mockRightConditionInput = mock(RexInputRef.class);
    when(mockLeftConditionInput.getIndex()).thenReturn(0);
    when(mockRightConditionInput.getIndex()).thenReturn(0);
    List<RexNode> condOperands = new ArrayList<>();
    condOperands.add(mockLeftConditionInput);
    condOperands.add(mockRightConditionInput);
    when(mockJoinCondition.getOperands()).thenReturn(condOperands);
    RelDataType mockLeftCondDataType = mock(RelDataType.class);
    RelDataType mockRightCondDataType = mock(RelDataType.class);
    when(mockLeftCondDataType.getSqlTypeName()).thenReturn(SqlTypeName.BOOLEAN);
    when(mockRightCondDataType.getSqlTypeName()).thenReturn(SqlTypeName.BOOLEAN);
    when(mockLeftConditionInput.getType()).thenReturn(mockLeftCondDataType);
    when(mockRightConditionInput.getType()).thenReturn(mockRightCondDataType);
    RelDataType mockLeftRowType = mock(RelDataType.class);
    // ?? why ??
    when(mockLeftRowType.getFieldCount()).thenReturn(0);
    when(mockLeftInput.getRowType()).thenReturn(mockLeftRowType);
    List<String> leftFieldNames = Collections.singletonList("test_table_field1");
    List<String> rightStreamFieldNames = Collections.singletonList("test_stream_field1");
    when(mockLeftRowType.getFieldNames()).thenReturn(leftFieldNames);
    RelDataType mockRightRowType = mock(RelDataType.class);
    when(mockRightInput.getRowType()).thenReturn(mockRightRowType);
    when(mockRightRowType.getFieldNames()).thenReturn(rightStreamFieldNames);
    StreamApplicationDescriptorImpl mockAppDesc = mock(StreamApplicationDescriptorImpl.class);
    OperatorSpec<Object, SamzaSqlRelMessage> mockLeftInputOp = mock(OperatorSpec.class);
    MessageStream<SamzaSqlRelMessage> mockLeftInputStream = new MessageStreamImpl<>(mockAppDesc, mockLeftInputOp);
    when(mockTranslatorContext.getMessageStream(eq(mockLeftInput.getId()))).thenReturn(mockLeftInputStream);
    OperatorSpec<Object, SamzaSqlRelMessage> mockRightInputOp = mock(OperatorSpec.class);
    MessageStream<SamzaSqlRelMessage> mockRightInputStream = new MessageStreamImpl<>(mockAppDesc, mockRightInputOp);
    when(mockTranslatorContext.getMessageStream(eq(mockRightInput.getId()))).thenReturn(mockRightInputStream);
    when(mockTranslatorContext.getStreamAppDescriptor()).thenReturn(mockAppDesc);
    InputOperatorSpec mockInputOp = mock(InputOperatorSpec.class);
    OutputStreamImpl mockOutputStream = mock(OutputStreamImpl.class);
    when(mockInputOp.isKeyed()).thenReturn(true);
    when(mockOutputStream.isKeyed()).thenReturn(true);
    doAnswer(this.getRegisterMessageStreamAnswer()).when(mockTranslatorContext).registerMessageStream(eq(3), any(MessageStream.class));
    RexToJavaCompiler mockCompiler = mock(RexToJavaCompiler.class);
    when(mockTranslatorContext.getExpressionCompiler()).thenReturn(mockCompiler);
    Expression mockExpr = mock(Expression.class);
    when(mockCompiler.compile(any(), any())).thenReturn(mockExpr);
    if (isRemoteTable) {
        doAnswer(this.getRegisteredTableAnswer()).when(mockAppDesc).getTable(any(RemoteTableDescriptor.class));
    } else {
        IntermediateMessageStreamImpl mockPartitionedStream = new IntermediateMessageStreamImpl(mockAppDesc, mockInputOp, mockOutputStream);
        when(mockAppDesc.getIntermediateStream(any(String.class), any(Serde.class), eq(false))).thenReturn(mockPartitionedStream);
        doAnswer(this.getRegisteredTableAnswer()).when(mockAppDesc).getTable(any(RocksDbTableDescriptor.class));
    }
    when(mockJoin.getJoinType()).thenReturn(JoinRelType.INNER);
    SamzaSqlExecutionContext mockExecutionContext = mock(SamzaSqlExecutionContext.class);
    when(mockTranslatorContext.getExecutionContext()).thenReturn(mockExecutionContext);
    SamzaSqlApplicationConfig mockAppConfig = mock(SamzaSqlApplicationConfig.class);
    when(mockExecutionContext.getSamzaSqlApplicationConfig()).thenReturn(mockAppConfig);
    Map<String, SqlIOConfig> ssConfigBySource = mock(HashMap.class);
    when(mockAppConfig.getInputSystemStreamConfigBySource()).thenReturn(ssConfigBySource);
    SqlIOConfig mockIOConfig = mock(SqlIOConfig.class);
    TableDescriptor mockTableDesc;
    if (isRemoteTable) {
        mockTableDesc = mock(RemoteTableDescriptor.class);
    } else {
        mockTableDesc = mock(RocksDbTableDescriptor.class);
    }
    when(ssConfigBySource.get(String.join(".", qualifiedTableName))).thenReturn(mockIOConfig);
    when(mockIOConfig.getTableDescriptor()).thenReturn(Optional.of(mockTableDesc));
    JoinTranslator joinTranslator = new JoinTranslator(logicalOpId, "", queryId);
    // Verify Metrics Works with Join
    Context mockContext = mock(Context.class);
    ContainerContext mockContainerContext = mock(ContainerContext.class);
    TestMetricsRegistryImpl testMetricsRegistryImpl = new TestMetricsRegistryImpl();
    when(mockContext.getContainerContext()).thenReturn(mockContainerContext);
    when(mockContainerContext.getContainerMetricsRegistry()).thenReturn(testMetricsRegistryImpl);
    TranslatorInputMetricsMapFunction inputMetricsMF = joinTranslator.getInputMetricsMF();
    assertNotNull(inputMetricsMF);
    inputMetricsMF.init(mockContext);
    TranslatorOutputMetricsMapFunction outputMetricsMF = joinTranslator.getOutputMetricsMF();
    assertNotNull(outputMetricsMF);
    outputMetricsMF.init(mockContext);
    assertEquals(1, testMetricsRegistryImpl.getCounters().size());
    assertEquals(2, testMetricsRegistryImpl.getCounters().get(logicalOpId).size());
    assertEquals(0, testMetricsRegistryImpl.getCounters().get(logicalOpId).get(0).getCount());
    assertEquals(0, testMetricsRegistryImpl.getCounters().get(logicalOpId).get(1).getCount());
    assertEquals(1, testMetricsRegistryImpl.getGauges().size());
    // Apply translate() method to verify that we are getting the correct map operator constructed
    joinTranslator.translate(mockJoin, mockTranslatorContext);
    // make sure that context has been registered with LogicFilter and output message streams
    verify(mockTranslatorContext, times(1)).registerMessageStream(3, this.getRegisteredMessageStream(3));
    when(mockTranslatorContext.getRelNode(3)).thenReturn(mockJoin);
    when(mockTranslatorContext.getMessageStream(3)).thenReturn(this.getRegisteredMessageStream(3));
    StreamTableJoinOperatorSpec joinSpec = (StreamTableJoinOperatorSpec) Whitebox.getInternalState(this.getRegisteredMessageStream(3), "operatorSpec");
    assertNotNull(joinSpec);
    assertEquals(joinSpec.getOpCode(), OperatorSpec.OpCode.JOIN);
    // Verify joinSpec has the corresponding setup
    StreamTableJoinFunction joinFn = joinSpec.getJoinFn();
    assertNotNull(joinFn);
    if (isRemoteTable) {
        assertTrue(joinFn instanceof SamzaSqlRemoteTableJoinFunction);
    } else {
        assertTrue(joinFn instanceof SamzaSqlLocalTableJoinFunction);
    }
    assertTrue(Whitebox.getInternalState(joinFn, "isTablePosOnRight").equals(false));
    assertEquals(Collections.singletonList(0), Whitebox.getInternalState(joinFn, "streamFieldIds"));
    assertEquals(leftFieldNames, Whitebox.getInternalState(joinFn, "tableFieldNames"));
    List<String> outputFieldNames = new ArrayList<>();
    outputFieldNames.addAll(leftFieldNames);
    outputFieldNames.addAll(rightStreamFieldNames);
    assertEquals(outputFieldNames, Whitebox.getInternalState(joinFn, "outFieldNames"));
}
Also used : Serde(org.apache.samza.serializers.Serde) IntermediateMessageStreamImpl(org.apache.samza.operators.stream.IntermediateMessageStreamImpl) MessageStreamImpl(org.apache.samza.operators.MessageStreamImpl) OutputStreamImpl(org.apache.samza.operators.spec.OutputStreamImpl) ArrayList(java.util.ArrayList) RelDataType(org.apache.calcite.rel.type.RelDataType) RexCall(org.apache.calcite.rex.RexCall) ContainerContext(org.apache.samza.context.ContainerContext) StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) RocksDbTableDescriptor(org.apache.samza.storage.kv.descriptors.RocksDbTableDescriptor) MessageStream(org.apache.samza.operators.MessageStream) StreamTableJoinFunction(org.apache.samza.operators.functions.StreamTableJoinFunction) SqlIOConfig(org.apache.samza.sql.interfaces.SqlIOConfig) SamzaSqlExecutionContext(org.apache.samza.sql.data.SamzaSqlExecutionContext) Context(org.apache.samza.context.Context) ContainerContext(org.apache.samza.context.ContainerContext) InputOperatorSpec(org.apache.samza.operators.spec.InputOperatorSpec) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) IntermediateMessageStreamImpl(org.apache.samza.operators.stream.IntermediateMessageStreamImpl) RexToJavaCompiler(org.apache.samza.sql.data.RexToJavaCompiler) TableDescriptor(org.apache.samza.table.descriptors.TableDescriptor) RocksDbTableDescriptor(org.apache.samza.storage.kv.descriptors.RocksDbTableDescriptor) RemoteTableDescriptor(org.apache.samza.table.descriptors.RemoteTableDescriptor) TestMetricsRegistryImpl(org.apache.samza.sql.util.TestMetricsRegistryImpl) RelNode(org.apache.calcite.rel.RelNode) Expression(org.apache.samza.sql.data.Expression) LogicalJoin(org.apache.calcite.rel.logical.LogicalJoin) RexInputRef(org.apache.calcite.rex.RexInputRef) SamzaSqlExecutionContext(org.apache.samza.sql.data.SamzaSqlExecutionContext) RelOptTable(org.apache.calcite.plan.RelOptTable) StreamTableJoinOperatorSpec(org.apache.samza.operators.spec.StreamTableJoinOperatorSpec) SamzaSqlRelMessage(org.apache.samza.sql.data.SamzaSqlRelMessage) RexNode(org.apache.calcite.rex.RexNode)

Example 8 with LogicalJoin

use of org.apache.calcite.rel.logical.LogicalJoin in project samza by apache.

the class TestQueryPlanner method testRemoteJoinWithUdfAndFilterHelper.

void testRemoteJoinWithUdfAndFilterHelper(boolean enableOptimizer) throws SamzaSqlValidatorException {
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(1);
    String sql = "Insert into testavro.enrichedPageViewTopic " + "select pv.pageKey as __key__, pv.pageKey as pageKey, coalesce(null, 'N/A') as companyName," + "       p.name as profileName, p.address as profileAddress " + "from testRemoteStore.Profile.`$table` as p " + "join testavro.PAGEVIEW as pv " + " on p.__key__ = BuildOutputRecord('id', pv.profileId)" + " where p.name = 'Mike' and pv.profileId = 1 and p.name = pv.pageKey";
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_ENABLE_PLAN_OPTIMIZER, Boolean.toString(enableOptimizer));
    Config samzaConfig = new MapConfig(staticConfigs);
    DslConverter dslConverter = new SamzaSqlDslConverterFactory().create(samzaConfig);
    Collection<RelRoot> relRoots = dslConverter.convertDsl(sql);
    /*
      Query plan without optimization:
      LogicalProject(__key__=[$9], pageKey=[$9], companyName=['N/A'], profileName=[$2], profileAddress=[$4])
        LogicalFilter(condition=[AND(=($2, 'Mike'), =($10, 1), =($2, $9))])  ==> Only the second condition could be pushed down.
          LogicalProject(__key__=[$0], id=[$1], name=[$2], companyId=[$3], address=[$4], selfEmployed=[$5],
                                  phoneNumbers=[$6], mapValues=[$7], __key__0=[$8], pageKey=[$9], profileId=[$10])
                                  ==> ProjectMergeRule removes this redundant node.
            LogicalJoin(condition=[=($0, $11)], joinType=[inner])
              LogicalTableScan(table=[[testRemoteStore, Profile, $table]])
              LogicalProject(__key__=[$0], pageKey=[$1], profileId=[$2], $f3=[BuildOutputRecord('id', $2)])  ==> Filter is pushed above project.
                LogicalTableScan(table=[[testavro, PAGEVIEW]])

      Query plan with optimization:
      LogicalProject(__key__=[$9], pageKey=[$9], companyName=['N/A'], profileName=[$2], profileAddress=[$4])
          LogicalFilter(condition=[AND(=($2, 'Mike'), =($2, $9))])
            LogicalJoin(condition=[=($0, $11)], joinType=[inner])
              LogicalTableScan(table=[[testRemoteStore, Profile, $table]])
              LogicalFilter(condition=[=($2, 1)])
                LogicalProject(__key__=[$0], pageKey=[$1], profileId=[$2], $f3=[BuildOutputRecord('id', $2)])
                  LogicalTableScan(table=[[testavro, PAGEVIEW]])
     */
    assertEquals(1, relRoots.size());
    RelRoot relRoot = relRoots.iterator().next();
    RelNode relNode = relRoot.rel;
    assertTrue(relNode instanceof LogicalProject);
    relNode = relNode.getInput(0);
    assertTrue(relNode instanceof LogicalFilter);
    if (enableOptimizer) {
        assertEquals("AND(=($2, $9), =($2, 'Mike'))", ((LogicalFilter) relNode).getCondition().toString());
    } else {
        assertEquals("AND(=($2, $9), =(1, $10), =($2, 'Mike'))", ((LogicalFilter) relNode).getCondition().toString());
    }
    relNode = relNode.getInput(0);
    if (enableOptimizer) {
        assertTrue(relNode instanceof LogicalJoin);
        assertEquals(2, relNode.getInputs().size());
    } else {
        assertTrue(relNode instanceof LogicalProject);
        relNode = relNode.getInput(0);
    }
    LogicalJoin join = (LogicalJoin) relNode;
    RelNode left = join.getLeft();
    RelNode right = join.getRight();
    assertTrue(left instanceof LogicalTableScan);
    if (enableOptimizer) {
        assertTrue(right instanceof LogicalFilter);
        assertEquals("=(1, $2)", ((LogicalFilter) right).getCondition().toString());
        relNode = right.getInput(0);
    } else {
        relNode = right;
    }
    assertTrue(relNode instanceof LogicalProject);
    relNode = relNode.getInput(0);
    assertTrue(relNode instanceof LogicalTableScan);
}
Also used : DslConverter(org.apache.samza.sql.interfaces.DslConverter) SamzaSqlTestConfig(org.apache.samza.sql.util.SamzaSqlTestConfig) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) LogicalFilter(org.apache.calcite.rel.logical.LogicalFilter) SamzaSqlDslConverterFactory(org.apache.samza.sql.dsl.SamzaSqlDslConverterFactory) RelRoot(org.apache.calcite.rel.RelRoot) LogicalTableScan(org.apache.calcite.rel.logical.LogicalTableScan) RelNode(org.apache.calcite.rel.RelNode) LogicalJoin(org.apache.calcite.rel.logical.LogicalJoin) MapConfig(org.apache.samza.config.MapConfig) LogicalProject(org.apache.calcite.rel.logical.LogicalProject)

Example 9 with LogicalJoin

use of org.apache.calcite.rel.logical.LogicalJoin in project samza by apache.

the class TestQueryPlanner method testRemoteJoinFilterPushDownWithUdfInFilterAndOptimizer.

@Test
public void testRemoteJoinFilterPushDownWithUdfInFilterAndOptimizer() throws SamzaSqlValidatorException {
    Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(1);
    String sql = "Insert into testavro.enrichedPageViewTopic " + "select pv.pageKey as __key__, pv.pageKey as pageKey, coalesce(null, 'N/A') as companyName," + "       p.name as profileName, p.address as profileAddress " + "from testRemoteStore.Profile.`$table` as p " + "join testavro.PAGEVIEW as pv " + " on p.__key__ = pv.profileId" + " where p.name = pv.pageKey AND p.name = 'Mike' AND pv.profileId = MyTest(pv.profileId)";
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
    staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_ENABLE_PLAN_OPTIMIZER, Boolean.toString(true));
    Config samzaConfig = new MapConfig(staticConfigs);
    DslConverter dslConverter = new SamzaSqlDslConverterFactory().create(samzaConfig);
    Collection<RelRoot> relRoots = dslConverter.convertDsl(sql);
    /*
      Query plan without optimization:
      LogicalProject(__key__=[$9], pageKey=[$9], companyName=['N/A'], profileName=[$2], profileAddress=[$4])
        LogicalFilter(condition=[AND(=($2, $9), =($2, 'Mike'), =($10, CAST(MyTest($10)):INTEGER))])
          LogicalJoin(condition=[=($0, $10)], joinType=[inner])
            LogicalTableScan(table=[[testRemoteStore, Profile, $table]])
            LogicalTableScan(table=[[testavro, PAGEVIEW]])

      Query plan with optimization:
      LogicalProject(__key__=[$9], pageKey=[$9], companyName=['N/A'], profileName=[$2], profileAddress=[$4])
        LogicalFilter(condition=[AND(=($2, $9), =($2, 'Mike'))])
          LogicalJoin(condition=[=($0, $10)], joinType=[inner])
            LogicalTableScan(table=[[testRemoteStore, Profile, $table]])
            LogicalFilter(condition=[=($2, CAST(MyTest($2)):INTEGER)])
              LogicalTableScan(table=[[testavro, PAGEVIEW]])
     */
    assertEquals(1, relRoots.size());
    RelRoot relRoot = relRoots.iterator().next();
    RelNode relNode = relRoot.rel;
    assertTrue(relNode instanceof LogicalProject);
    relNode = relNode.getInput(0);
    assertTrue(relNode instanceof LogicalFilter);
    assertEquals("AND(=($2, $9), =($2, 'Mike'))", ((LogicalFilter) relNode).getCondition().toString());
    relNode = relNode.getInput(0);
    assertTrue(relNode instanceof LogicalJoin);
    assertEquals(2, relNode.getInputs().size());
    LogicalJoin join = (LogicalJoin) relNode;
    RelNode left = join.getLeft();
    RelNode right = join.getRight();
    assertTrue(left instanceof LogicalTableScan);
    assertTrue(right instanceof LogicalFilter);
    assertEquals("=($2, CAST(MyTest($2)):INTEGER)", ((LogicalFilter) right).getCondition().toString());
    assertTrue(right.getInput(0) instanceof LogicalTableScan);
}
Also used : DslConverter(org.apache.samza.sql.interfaces.DslConverter) SamzaSqlTestConfig(org.apache.samza.sql.util.SamzaSqlTestConfig) SamzaSqlApplicationConfig(org.apache.samza.sql.runner.SamzaSqlApplicationConfig) Config(org.apache.samza.config.Config) MapConfig(org.apache.samza.config.MapConfig) LogicalFilter(org.apache.calcite.rel.logical.LogicalFilter) SamzaSqlDslConverterFactory(org.apache.samza.sql.dsl.SamzaSqlDslConverterFactory) RelRoot(org.apache.calcite.rel.RelRoot) LogicalTableScan(org.apache.calcite.rel.logical.LogicalTableScan) RelNode(org.apache.calcite.rel.RelNode) LogicalJoin(org.apache.calcite.rel.logical.LogicalJoin) MapConfig(org.apache.samza.config.MapConfig) LogicalProject(org.apache.calcite.rel.logical.LogicalProject) Test(org.junit.Test)

Example 10 with LogicalJoin

use of org.apache.calcite.rel.logical.LogicalJoin in project flink by apache.

the class FlinkSemiAntiJoinProjectTransposeRule method onMatch.

// ~ Methods ----------------------------------------------------------------
public void onMatch(RelOptRuleCall call) {
    LogicalJoin join = call.rel(0);
    LogicalProject project = call.rel(1);
    // convert the semi/anti join condition to reflect the LHS with the project
    // pulled up
    RexNode newCondition = adjustCondition(project, join);
    Join newJoin = LogicalJoin.create(project.getInput(), join.getRight(), join.getHints(), newCondition, join.getVariablesSet(), join.getJoinType());
    // Create the new projection. Note that the projection expressions
    // are the same as the original because they only reference the LHS
    // of the semi/anti join and the semi/anti join only projects out the LHS
    final RelBuilder relBuilder = call.builder();
    relBuilder.push(newJoin);
    relBuilder.project(project.getProjects(), project.getRowType().getFieldNames());
    call.transformTo(relBuilder.build());
}
Also used : RelBuilder(org.apache.calcite.tools.RelBuilder) LogicalJoin(org.apache.calcite.rel.logical.LogicalJoin) Join(org.apache.calcite.rel.core.Join) LogicalJoin(org.apache.calcite.rel.logical.LogicalJoin) LogicalProject(org.apache.calcite.rel.logical.LogicalProject) RexNode(org.apache.calcite.rex.RexNode)

Aggregations

LogicalJoin (org.apache.calcite.rel.logical.LogicalJoin)27 RelNode (org.apache.calcite.rel.RelNode)20 RexNode (org.apache.calcite.rex.RexNode)14 LogicalProject (org.apache.calcite.rel.logical.LogicalProject)13 ArrayList (java.util.ArrayList)8 LogicalFilter (org.apache.calcite.rel.logical.LogicalFilter)8 TableScan (org.apache.calcite.rel.core.TableScan)5 LogicalTableScan (org.apache.calcite.rel.logical.LogicalTableScan)5 RelDataType (org.apache.calcite.rel.type.RelDataType)5 RelTraitSet (org.apache.calcite.plan.RelTraitSet)4 InvalidRelException (org.apache.calcite.rel.InvalidRelException)4 Join (org.apache.calcite.rel.core.Join)4 RelDataTypeField (org.apache.calcite.rel.type.RelDataTypeField)4 RexBuilder (org.apache.calcite.rex.RexBuilder)4 RexInputRef (org.apache.calcite.rex.RexInputRef)4 ImmutableBitSet (org.apache.calcite.util.ImmutableBitSet)4 RelOptCluster (org.apache.calcite.plan.RelOptCluster)3 RelRoot (org.apache.calcite.rel.RelRoot)3 JoinRelType (org.apache.calcite.rel.core.JoinRelType)3 LogicalAggregate (org.apache.calcite.rel.logical.LogicalAggregate)3