Search in sources :

Example 1 with JOIN_DISTRIBUTION_TYPE

use of io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE in project hetu-core by openlookeng.

the class TestReorderJoins method testReplicatesWhenNotRestricted.

@Test
public void testReplicatesWhenNotRestricted() {
    // variable width so that average row size is respected
    Type symbolType = createUnboundedVarcharType();
    int aRows = 10_000;
    int bRows = 10;
    PlanNodeStatsEstimate probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
    PlanNodeStatsEstimate buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
    // B table is small enough to be replicated in AUTOMATIC_RESTRICTED mode
    assertReorderJoins().setSystemProperty(JOIN_DISTRIBUTION_TYPE, AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").on(p -> {
        Symbol a1 = p.symbol("A1", symbolType);
        Symbol b1 = p.symbol("B1", symbolType);
        return p.join(INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty());
    }).overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).matches(join(INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
    probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
    buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
    // B table exceeds AUTOMATIC_RESTRICTED limit therefore it is partitioned
    assertReorderJoins().setSystemProperty(JOIN_DISTRIBUTION_TYPE, AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").on(p -> {
        Symbol a1 = p.symbol("A1", symbolType);
        Symbol b1 = p.symbol("B1", symbolType);
        return p.join(INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), ImmutableList.of(new EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty());
    }).overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).matches(join(INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(PARTITIONED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
Also used : SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) REPLICATED(io.prestosql.spi.plan.JoinNode.DistributionType.REPLICATED) EquiJoinClause(io.prestosql.spi.plan.JoinNode.EquiJoinClause) PlanMatchPattern.equiJoinClause(io.prestosql.sql.planner.assertions.PlanMatchPattern.equiJoinClause) QualifiedName(io.prestosql.sql.tree.QualifiedName) BROADCAST(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType.BROADCAST) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) Test(org.testng.annotations.Test) PARTITIONED(io.prestosql.spi.plan.JoinNode.DistributionType.PARTITIONED) LESS_THAN(io.prestosql.spi.function.OperatorType.LESS_THAN) JOIN_REORDERING_STRATEGY(io.prestosql.SystemSessionProperties.JOIN_REORDERING_STRATEGY) PlanMatchPattern.values(io.prestosql.sql.planner.assertions.PlanMatchPattern.values) ImmutableList(com.google.common.collect.ImmutableList) Expressions.call(io.prestosql.sql.relational.Expressions.call) OperatorType(io.prestosql.spi.function.OperatorType) JoinReorderingStrategy(io.prestosql.sql.analyzer.FeaturesConfig.JoinReorderingStrategy) BOOLEAN(io.prestosql.spi.type.BooleanType.BOOLEAN) Type(io.prestosql.spi.type.Type) RuleAssert(io.prestosql.sql.planner.iterative.rule.test.RuleAssert) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) Symbol(io.prestosql.spi.plan.Symbol) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) AfterClass(org.testng.annotations.AfterClass) EQUAL(io.prestosql.spi.function.OperatorType.EQUAL) PlanMatchPattern.join(io.prestosql.sql.planner.assertions.PlanMatchPattern.join) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) ImmutableMap(com.google.common.collect.ImmutableMap) PlanMatchPattern(io.prestosql.sql.planner.assertions.PlanMatchPattern) BeforeClass(org.testng.annotations.BeforeClass) FunctionAndTypeManager.qualifyObjectName(io.prestosql.metadata.FunctionAndTypeManager.qualifyObjectName) CostComparator(io.prestosql.cost.CostComparator) AUTOMATIC(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType.AUTOMATIC) Expressions.variable(io.prestosql.sql.relational.Expressions.variable) List(java.util.List) Closeables.closeAllRuntimeException(io.airlift.testing.Closeables.closeAllRuntimeException) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) RowExpression(io.prestosql.spi.relation.RowExpression) INNER(io.prestosql.spi.plan.JoinNode.Type.INNER) Optional(java.util.Optional) JOIN_MAX_BROADCAST_TABLE_SIZE(io.prestosql.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE) RuleTester(io.prestosql.sql.planner.iterative.rule.test.RuleTester) JOIN_DISTRIBUTION_TYPE(io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) OperatorType(io.prestosql.spi.function.OperatorType) Type(io.prestosql.spi.type.Type) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) Symbol(io.prestosql.spi.plan.Symbol) EquiJoinClause(io.prestosql.spi.plan.JoinNode.EquiJoinClause) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) Test(org.testng.annotations.Test)

Example 2 with JOIN_DISTRIBUTION_TYPE

use of io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE in project hetu-core by openlookeng.

the class TestLogicalPlanner method testBroadcastCorrelatedSubqueryAvoidsRemoteExchangeBeforeAggregation.

@Test
public void testBroadcastCorrelatedSubqueryAvoidsRemoteExchangeBeforeAggregation() {
    Session broadcastJoin = Session.builder(this.getQueryRunner().getDefaultSession()).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.BROADCAST.name()).setSystemProperty(FORCE_SINGLE_NODE_OUTPUT, Boolean.toString(false)).build();
    // make sure there is a remote exchange on the build side
    PlanMatchPattern joinBuildSideWithRemoteExchange = anyTree(node(JoinNode.class, anyTree(node(TableScanNode.class)), anyTree(exchange(REMOTE, REPLICATE, anyTree(node(TableScanNode.class))))));
    // validates that there exists only one remote exchange
    Consumer<Plan> validateSingleRemoteExchange = plan -> assertEquals(countOfMatchingNodes(plan, node -> node instanceof ExchangeNode && ((ExchangeNode) node).getScope() == REMOTE), 1);
    Consumer<Plan> validateSingleStreamingAggregation = plan -> assertEquals(countOfMatchingNodes(plan, node -> node instanceof AggregationNode && ((AggregationNode) node).getGroupingKeys().contains(new Symbol("unique")) && ((AggregationNode) node).isStreamable()), 1);
    // region is unpartitioned, AssignUniqueId should provide satisfying partitioning for count(*) after LEFT JOIN
    assertPlanWithSession("SELECT (SELECT count(*) FROM region r2 WHERE r2.regionkey > r1.regionkey) FROM region r1", broadcastJoin, false, joinBuildSideWithRemoteExchange, validateSingleRemoteExchange.andThen(validateSingleStreamingAggregation));
    // orders is naturally partitioned, AssignUniqueId should not overwrite its natural partitioning
    assertPlanWithSession("SELECT count(count) " + "FROM (SELECT o1.orderkey orderkey, (SELECT count(*) FROM orders o2 WHERE o2.orderkey > o1.orderkey) count FROM orders o1) " + "GROUP BY orderkey", broadcastJoin, false, joinBuildSideWithRemoteExchange, validateSingleRemoteExchange.andThen(validateSingleStreamingAggregation));
}
Also used : REPLICATED(io.prestosql.spi.plan.JoinNode.DistributionType.REPLICATED) SortNode(io.prestosql.sql.planner.plan.SortNode) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) OPTIMIZE_HASH_GENERATION(io.prestosql.SystemSessionProperties.OPTIMIZE_HASH_GENERATION) PlanMatchPattern.markDistinct(io.prestosql.sql.planner.assertions.PlanMatchPattern.markDistinct) ValueSet(io.prestosql.spi.predicate.ValueSet) Test(org.testng.annotations.Test) PlanMatchPattern.singleGroupingSet(io.prestosql.sql.planner.assertions.PlanMatchPattern.singleGroupingSet) AggregationNode(io.prestosql.spi.plan.AggregationNode) JOIN_REORDERING_STRATEGY(io.prestosql.SystemSessionProperties.JOIN_REORDERING_STRATEGY) PlanMatchPattern.values(io.prestosql.sql.planner.assertions.PlanMatchPattern.values) Slices(io.airlift.slice.Slices) Map(java.util.Map) Domain.singleValue(io.prestosql.spi.predicate.Domain.singleValue) PlanMatchPattern.node(io.prestosql.sql.planner.assertions.PlanMatchPattern.node) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) PlanMatchPattern.strictTableScan(io.prestosql.sql.planner.assertions.PlanMatchPattern.strictTableScan) PlanMatchPattern.expression(io.prestosql.sql.planner.assertions.PlanMatchPattern.expression) Assert.assertFalse(org.testng.Assert.assertFalse) PlanMatchPattern.join(io.prestosql.sql.planner.assertions.PlanMatchPattern.join) PlanMatchPattern.strictProject(io.prestosql.sql.planner.assertions.PlanMatchPattern.strictProject) PlanMatchPattern(io.prestosql.sql.planner.assertions.PlanMatchPattern) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TableScanNode(io.prestosql.spi.plan.TableScanNode) PlanNode(io.prestosql.spi.plan.PlanNode) ProjectNode(io.prestosql.spi.plan.ProjectNode) VarcharType.createVarcharType(io.prestosql.spi.type.VarcharType.createVarcharType) MorePredicates(io.prestosql.util.MorePredicates) CheckSubqueryNodesAreRewritten(io.prestosql.sql.planner.optimizations.CheckSubqueryNodesAreRewritten) LongLiteral(io.prestosql.sql.tree.LongLiteral) QueryTemplate(io.prestosql.tests.QueryTemplate) LOCAL(io.prestosql.sql.planner.plan.ExchangeNode.Scope.LOCAL) Domain(io.prestosql.spi.predicate.Domain) INNER(io.prestosql.spi.plan.JoinNode.Type.INNER) StatisticsWriterNode(io.prestosql.sql.planner.plan.StatisticsWriterNode) DistinctLimitNode(io.prestosql.sql.planner.plan.DistinctLimitNode) OPTIMIZED(io.prestosql.sql.planner.LogicalPlanner.Stage.OPTIMIZED) JOIN_DISTRIBUTION_TYPE(io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) QueryTemplate.queryTemplate(io.prestosql.tests.QueryTemplate.queryTemplate) PlanMatchPattern.equiJoinClause(io.prestosql.sql.planner.assertions.PlanMatchPattern.equiJoinClause) PlanMatchPattern.assignUniqueId(io.prestosql.sql.planner.assertions.PlanMatchPattern.assignUniqueId) REMOTE(io.prestosql.sql.planner.plan.ExchangeNode.Scope.REMOTE) GATHER(io.prestosql.sql.planner.plan.ExchangeNode.Type.GATHER) SINGLE(io.prestosql.spi.plan.AggregationNode.Step.SINGLE) REPARTITION(io.prestosql.sql.planner.plan.ExchangeNode.Type.REPARTITION) Session(io.prestosql.Session) PlanMatchPattern.anyTree(io.prestosql.sql.planner.assertions.PlanMatchPattern.anyTree) MoreCollectors.toOptional(com.google.common.collect.MoreCollectors.toOptional) PlanMatchPattern.output(io.prestosql.sql.planner.assertions.PlanMatchPattern.output) ExpressionMatcher(io.prestosql.sql.planner.assertions.ExpressionMatcher) PlanMatchPattern.anyNot(io.prestosql.sql.planner.assertions.PlanMatchPattern.anyNot) ValuesNode(io.prestosql.spi.plan.ValuesNode) DESCENDING(io.prestosql.sql.tree.SortItem.Ordering.DESCENDING) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) LimitNode(io.prestosql.spi.plan.LimitNode) PlanOptimizer(io.prestosql.sql.planner.optimizations.PlanOptimizer) PlanMatchPattern.any(io.prestosql.sql.planner.assertions.PlanMatchPattern.any) PlanMatchPattern.aggregation(io.prestosql.sql.planner.assertions.PlanMatchPattern.aggregation) PlanMatchPattern.project(io.prestosql.sql.planner.assertions.PlanMatchPattern.project) PlanMatchPattern.tableScan(io.prestosql.sql.planner.assertions.PlanMatchPattern.tableScan) TpchColumnHandle(io.prestosql.plugin.tpch.TpchColumnHandle) PlanMatchPattern.semiJoin(io.prestosql.sql.planner.assertions.PlanMatchPattern.semiJoin) RowNumberSymbolMatcher(io.prestosql.sql.planner.assertions.RowNumberSymbolMatcher) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) FilterNode(io.prestosql.spi.plan.FilterNode) JoinReorderingStrategy(io.prestosql.sql.analyzer.FeaturesConfig.JoinReorderingStrategy) FORCE_SINGLE_NODE_OUTPUT(io.prestosql.SystemSessionProperties.FORCE_SINGLE_NODE_OUTPUT) ASC_NULLS_LAST(io.prestosql.spi.block.SortOrder.ASC_NULLS_LAST) ApplyNode(io.prestosql.sql.planner.plan.ApplyNode) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) FINAL(io.prestosql.spi.plan.AggregationNode.Step.FINAL) IndexJoinNode(io.prestosql.sql.planner.plan.IndexJoinNode) FILTERING_SEMI_JOIN_TO_INNER(io.prestosql.SystemSessionProperties.FILTERING_SEMI_JOIN_TO_INNER) String.format(java.lang.String.format) PlanMatchPattern.sort(io.prestosql.sql.planner.assertions.PlanMatchPattern.sort) List(java.util.List) EnforceSingleRowNode(io.prestosql.sql.planner.plan.EnforceSingleRowNode) MorePredicates.isInstanceOfAny(io.prestosql.util.MorePredicates.isInstanceOfAny) TopNNode(io.prestosql.spi.plan.TopNNode) Entry(java.util.Map.Entry) Optional(java.util.Optional) PlanMatchPattern.topNRankingNumber(io.prestosql.sql.planner.assertions.PlanMatchPattern.topNRankingNumber) PlanMatchPattern.topN(io.prestosql.sql.planner.assertions.PlanMatchPattern.topN) PlanMatchPattern.constrainedTableScan(io.prestosql.sql.planner.assertions.PlanMatchPattern.constrainedTableScan) LAST(io.prestosql.sql.tree.SortItem.NullOrdering.LAST) TpchTableHandle(io.prestosql.plugin.tpch.TpchTableHandle) LateralJoinNode(io.prestosql.sql.planner.plan.LateralJoinNode) PlanMatchPattern.apply(io.prestosql.sql.planner.assertions.PlanMatchPattern.apply) Assert.assertEquals(org.testng.Assert.assertEquals) PARTITIONED(io.prestosql.spi.plan.JoinNode.DistributionType.PARTITIONED) PARTIAL(io.prestosql.spi.plan.AggregationNode.Step.PARTIAL) SemiJoinNode(io.prestosql.sql.planner.plan.SemiJoinNode) SUBQUERY_MULTIPLE_ROWS(io.prestosql.spi.StandardErrorCode.SUBQUERY_MULTIPLE_ROWS) ImmutableList(com.google.common.collect.ImmutableList) Range(io.prestosql.spi.predicate.Range) PlanMatchPattern.functionCall(io.prestosql.sql.planner.assertions.PlanMatchPattern.functionCall) PlanMatchPattern.filter(io.prestosql.sql.planner.assertions.PlanMatchPattern.filter) PlanMatchPattern.exchange(io.prestosql.sql.planner.assertions.PlanMatchPattern.exchange) REPLICATE(io.prestosql.sql.planner.plan.ExchangeNode.Type.REPLICATE) JoinNode(io.prestosql.spi.plan.JoinNode) Symbol(io.prestosql.spi.plan.Symbol) BasePlanTest(io.prestosql.sql.planner.assertions.BasePlanTest) ASCENDING(io.prestosql.sql.tree.SortItem.Ordering.ASCENDING) DISTRIBUTED_SORT(io.prestosql.SystemSessionProperties.DISTRIBUTED_SORT) TupleDomain(io.prestosql.spi.predicate.TupleDomain) PlanMatchPattern.limit(io.prestosql.sql.planner.assertions.PlanMatchPattern.limit) Consumer(java.util.function.Consumer) PlanNodeSearcher.searchFrom(io.prestosql.sql.planner.optimizations.PlanNodeSearcher.searchFrom) PlanMatchPattern.rowNumber(io.prestosql.sql.planner.assertions.PlanMatchPattern.rowNumber) AddLocalExchanges(io.prestosql.sql.planner.optimizations.AddLocalExchanges) LEFT(io.prestosql.spi.plan.JoinNode.Type.LEFT) PlanMatchPattern.constrainedTableScanWithTableLayout(io.prestosql.sql.planner.assertions.PlanMatchPattern.constrainedTableScanWithTableLayout) TableScanNode(io.prestosql.spi.plan.TableScanNode) ExchangeNode(io.prestosql.sql.planner.plan.ExchangeNode) IndexJoinNode(io.prestosql.sql.planner.plan.IndexJoinNode) LateralJoinNode(io.prestosql.sql.planner.plan.LateralJoinNode) SemiJoinNode(io.prestosql.sql.planner.plan.SemiJoinNode) JoinNode(io.prestosql.spi.plan.JoinNode) Symbol(io.prestosql.spi.plan.Symbol) PlanMatchPattern(io.prestosql.sql.planner.assertions.PlanMatchPattern) AggregationNode(io.prestosql.spi.plan.AggregationNode) Session(io.prestosql.Session) Test(org.testng.annotations.Test) BasePlanTest(io.prestosql.sql.planner.assertions.BasePlanTest)

Example 3 with JOIN_DISTRIBUTION_TYPE

use of io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE in project hetu-core by openlookeng.

the class TestDetermineJoinDistributionType method testReplicateLeftOuterJoin.

@Test
public void testReplicateLeftOuterJoin() {
    int aRows = 10_000;
    int bRows = 10;
    assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build()).on(p -> p.join(LEFT, p.values(new PlanNodeId("valuesA"), aRows, p.symbol("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.symbol("B1", BIGINT)), ImmutableList.of(new JoinNode.EquiJoinClause(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT))), ImmutableList.of(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
Also used : CostComparator(io.prestosql.cost.CostComparator) PlanBuilder.constantExpressions(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) REPLICATED(io.prestosql.spi.plan.JoinNode.DistributionType.REPLICATED) PlanMatchPattern.equiJoinClause(io.prestosql.sql.planner.assertions.PlanMatchPattern.equiJoinClause) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) PlanMatchPattern.enforceSingleRow(io.prestosql.sql.planner.assertions.PlanMatchPattern.enforceSingleRow) Test(org.testng.annotations.Test) PARTITIONED(io.prestosql.spi.plan.JoinNode.DistributionType.PARTITIONED) PlanBuilder.castToRowExpression(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.castToRowExpression) PlanMatchPattern.values(io.prestosql.sql.planner.assertions.PlanMatchPattern.values) ImmutableList(com.google.common.collect.ImmutableList) RuleAssert(io.prestosql.sql.planner.iterative.rule.test.RuleAssert) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) Type(io.prestosql.spi.plan.JoinNode.Type) JoinNode(io.prestosql.spi.plan.JoinNode) TaskCountEstimator(io.prestosql.cost.TaskCountEstimator) Symbol(io.prestosql.spi.plan.Symbol) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) AfterClass(org.testng.annotations.AfterClass) PlanMatchPattern.join(io.prestosql.sql.planner.assertions.PlanMatchPattern.join) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) ImmutableMap(com.google.common.collect.ImmutableMap) FULL(io.prestosql.spi.plan.JoinNode.Type.FULL) BeforeClass(org.testng.annotations.BeforeClass) CostComparator(io.prestosql.cost.CostComparator) RIGHT(io.prestosql.spi.plan.JoinNode.Type.RIGHT) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) DistributionType(io.prestosql.spi.plan.JoinNode.DistributionType) INNER(io.prestosql.spi.plan.JoinNode.Type.INNER) Optional(java.util.Optional) JOIN_MAX_BROADCAST_TABLE_SIZE(io.prestosql.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE) RuleTester(io.prestosql.sql.planner.iterative.rule.test.RuleTester) VarcharType(io.prestosql.spi.type.VarcharType) JOIN_DISTRIBUTION_TYPE(io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) LEFT(io.prestosql.spi.plan.JoinNode.Type.LEFT) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Symbol(io.prestosql.spi.plan.Symbol) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) Test(org.testng.annotations.Test)

Example 4 with JOIN_DISTRIBUTION_TYPE

use of io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE in project hetu-core by openlookeng.

the class TestDetermineJoinDistributionType method testFlipAndReplicateRightOuterJoin.

@Test
public void testFlipAndReplicateRightOuterJoin() {
    int aRows = 10;
    int bRows = 1_000_000;
    assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build()).on(p -> p.join(RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.symbol("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.symbol("B1", BIGINT)), ImmutableList.of(new JoinNode.EquiJoinClause(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT))), ImmutableList.of(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
Also used : CostComparator(io.prestosql.cost.CostComparator) PlanBuilder.constantExpressions(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) REPLICATED(io.prestosql.spi.plan.JoinNode.DistributionType.REPLICATED) PlanMatchPattern.equiJoinClause(io.prestosql.sql.planner.assertions.PlanMatchPattern.equiJoinClause) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) PlanMatchPattern.enforceSingleRow(io.prestosql.sql.planner.assertions.PlanMatchPattern.enforceSingleRow) Test(org.testng.annotations.Test) PARTITIONED(io.prestosql.spi.plan.JoinNode.DistributionType.PARTITIONED) PlanBuilder.castToRowExpression(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.castToRowExpression) PlanMatchPattern.values(io.prestosql.sql.planner.assertions.PlanMatchPattern.values) ImmutableList(com.google.common.collect.ImmutableList) RuleAssert(io.prestosql.sql.planner.iterative.rule.test.RuleAssert) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) Type(io.prestosql.spi.plan.JoinNode.Type) JoinNode(io.prestosql.spi.plan.JoinNode) TaskCountEstimator(io.prestosql.cost.TaskCountEstimator) Symbol(io.prestosql.spi.plan.Symbol) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) AfterClass(org.testng.annotations.AfterClass) PlanMatchPattern.join(io.prestosql.sql.planner.assertions.PlanMatchPattern.join) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) ImmutableMap(com.google.common.collect.ImmutableMap) FULL(io.prestosql.spi.plan.JoinNode.Type.FULL) BeforeClass(org.testng.annotations.BeforeClass) CostComparator(io.prestosql.cost.CostComparator) RIGHT(io.prestosql.spi.plan.JoinNode.Type.RIGHT) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) DistributionType(io.prestosql.spi.plan.JoinNode.DistributionType) INNER(io.prestosql.spi.plan.JoinNode.Type.INNER) Optional(java.util.Optional) JOIN_MAX_BROADCAST_TABLE_SIZE(io.prestosql.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE) RuleTester(io.prestosql.sql.planner.iterative.rule.test.RuleTester) VarcharType(io.prestosql.spi.type.VarcharType) JOIN_DISTRIBUTION_TYPE(io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) LEFT(io.prestosql.spi.plan.JoinNode.Type.LEFT) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Symbol(io.prestosql.spi.plan.Symbol) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) Test(org.testng.annotations.Test)

Example 5 with JOIN_DISTRIBUTION_TYPE

use of io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE in project hetu-core by openlookeng.

the class TestDetermineSemiJoinDistributionType method testReplicatesWhenNotRestricted.

@Test
public void testReplicatesWhenNotRestricted() {
    // variable width so that average row size is respected
    Type symbolType = createUnboundedVarcharType();
    int aRows = 10_000;
    int bRows = 10;
    PlanNodeStatsEstimate probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
    PlanNodeStatsEstimate buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
    // B table is small enough to be replicated in AUTOMATIC_RESTRICTED mode
    assertDetermineSemiJoinDistributionType().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).on(p -> {
        Symbol a1 = p.symbol("A1", symbolType);
        Symbol b1 = p.symbol("B1", symbolType);
        return p.semiJoin(p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), a1, b1, p.symbol("output"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
    }).matches(semiJoin("A1", "B1", "output", Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
    probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
    buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
    // B table exceeds AUTOMATIC_RESTRICTED limit therefore it is partitioned
    assertDetermineSemiJoinDistributionType().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).on(p -> {
        Symbol a1 = p.symbol("A1", symbolType);
        Symbol b1 = p.symbol("B1", symbolType);
        return p.semiJoin(p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), a1, b1, p.symbol("output"), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty());
    }).matches(semiJoin("A1", "B1", "output", Optional.of(PARTITIONED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
Also used : PlanBuilder.constantExpressions(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) REPLICATED(io.prestosql.sql.planner.plan.SemiJoinNode.DistributionType.REPLICATED) PlanMatchPattern.semiJoin(io.prestosql.sql.planner.assertions.PlanMatchPattern.semiJoin) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) Test(org.testng.annotations.Test) PlanMatchPattern.values(io.prestosql.sql.planner.assertions.PlanMatchPattern.values) ImmutableList(com.google.common.collect.ImmutableList) Type(io.prestosql.spi.type.Type) RuleAssert(io.prestosql.sql.planner.iterative.rule.test.RuleAssert) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) TaskCountEstimator(io.prestosql.cost.TaskCountEstimator) Symbol(io.prestosql.spi.plan.Symbol) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) AfterClass(org.testng.annotations.AfterClass) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) ImmutableMap(com.google.common.collect.ImmutableMap) BeforeClass(org.testng.annotations.BeforeClass) CostComparator(io.prestosql.cost.CostComparator) PARTITIONED(io.prestosql.sql.planner.plan.SemiJoinNode.DistributionType.PARTITIONED) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) Optional(java.util.Optional) JOIN_MAX_BROADCAST_TABLE_SIZE(io.prestosql.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE) RuleTester(io.prestosql.sql.planner.iterative.rule.test.RuleTester) JOIN_DISTRIBUTION_TYPE(io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) Type(io.prestosql.spi.type.Type) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) Symbol(io.prestosql.spi.plan.Symbol) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) Test(org.testng.annotations.Test)

Aggregations

JOIN_DISTRIBUTION_TYPE (io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE)21 Test (org.testng.annotations.Test)21 ImmutableList (com.google.common.collect.ImmutableList)17 ImmutableMap (com.google.common.collect.ImmutableMap)17 Symbol (io.prestosql.spi.plan.Symbol)17 JoinDistributionType (io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType)17 PlanMatchPattern.values (io.prestosql.sql.planner.assertions.PlanMatchPattern.values)17 Optional (java.util.Optional)17 JOIN_MAX_BROADCAST_TABLE_SIZE (io.prestosql.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE)16 CostComparator (io.prestosql.cost.CostComparator)16 PlanNodeStatsEstimate (io.prestosql.cost.PlanNodeStatsEstimate)16 SymbolStatsEstimate (io.prestosql.cost.SymbolStatsEstimate)16 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)16 BIGINT (io.prestosql.spi.type.BigintType.BIGINT)16 VarcharType.createUnboundedVarcharType (io.prestosql.spi.type.VarcharType.createUnboundedVarcharType)16 RuleAssert (io.prestosql.sql.planner.iterative.rule.test.RuleAssert)16 RuleTester (io.prestosql.sql.planner.iterative.rule.test.RuleTester)16 AfterClass (org.testng.annotations.AfterClass)16 BeforeClass (org.testng.annotations.BeforeClass)16 TaskCountEstimator (io.prestosql.cost.TaskCountEstimator)14