Search in sources :

Example 1 with CostComparator

use of io.prestosql.cost.CostComparator in project hetu-core by openlookeng.

the class TestDetermineJoinDistributionType method testReplicateLeftOuterJoin.

@Test
public void testReplicateLeftOuterJoin() {
    int aRows = 10_000;
    int bRows = 10;
    assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build()).on(p -> p.join(LEFT, p.values(new PlanNodeId("valuesA"), aRows, p.symbol("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.symbol("B1", BIGINT)), ImmutableList.of(new JoinNode.EquiJoinClause(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT))), ImmutableList.of(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
Also used : CostComparator(io.prestosql.cost.CostComparator) PlanBuilder.constantExpressions(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) REPLICATED(io.prestosql.spi.plan.JoinNode.DistributionType.REPLICATED) PlanMatchPattern.equiJoinClause(io.prestosql.sql.planner.assertions.PlanMatchPattern.equiJoinClause) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) PlanMatchPattern.enforceSingleRow(io.prestosql.sql.planner.assertions.PlanMatchPattern.enforceSingleRow) Test(org.testng.annotations.Test) PARTITIONED(io.prestosql.spi.plan.JoinNode.DistributionType.PARTITIONED) PlanBuilder.castToRowExpression(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.castToRowExpression) PlanMatchPattern.values(io.prestosql.sql.planner.assertions.PlanMatchPattern.values) ImmutableList(com.google.common.collect.ImmutableList) RuleAssert(io.prestosql.sql.planner.iterative.rule.test.RuleAssert) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) Type(io.prestosql.spi.plan.JoinNode.Type) JoinNode(io.prestosql.spi.plan.JoinNode) TaskCountEstimator(io.prestosql.cost.TaskCountEstimator) Symbol(io.prestosql.spi.plan.Symbol) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) AfterClass(org.testng.annotations.AfterClass) PlanMatchPattern.join(io.prestosql.sql.planner.assertions.PlanMatchPattern.join) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) ImmutableMap(com.google.common.collect.ImmutableMap) FULL(io.prestosql.spi.plan.JoinNode.Type.FULL) BeforeClass(org.testng.annotations.BeforeClass) CostComparator(io.prestosql.cost.CostComparator) RIGHT(io.prestosql.spi.plan.JoinNode.Type.RIGHT) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) DistributionType(io.prestosql.spi.plan.JoinNode.DistributionType) INNER(io.prestosql.spi.plan.JoinNode.Type.INNER) Optional(java.util.Optional) JOIN_MAX_BROADCAST_TABLE_SIZE(io.prestosql.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE) RuleTester(io.prestosql.sql.planner.iterative.rule.test.RuleTester) VarcharType(io.prestosql.spi.type.VarcharType) JOIN_DISTRIBUTION_TYPE(io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) LEFT(io.prestosql.spi.plan.JoinNode.Type.LEFT) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Symbol(io.prestosql.spi.plan.Symbol) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) Test(org.testng.annotations.Test)

Example 2 with CostComparator

use of io.prestosql.cost.CostComparator in project hetu-core by openlookeng.

the class TestDetermineJoinDistributionType method testFlipAndReplicateRightOuterJoin.

@Test
public void testFlipAndReplicateRightOuterJoin() {
    int aRows = 10;
    int bRows = 1_000_000;
    assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build()).on(p -> p.join(RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.symbol("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.symbol("B1", BIGINT)), ImmutableList.of(new JoinNode.EquiJoinClause(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT))), ImmutableList.of(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
Also used : CostComparator(io.prestosql.cost.CostComparator) PlanBuilder.constantExpressions(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) REPLICATED(io.prestosql.spi.plan.JoinNode.DistributionType.REPLICATED) PlanMatchPattern.equiJoinClause(io.prestosql.sql.planner.assertions.PlanMatchPattern.equiJoinClause) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) PlanMatchPattern.enforceSingleRow(io.prestosql.sql.planner.assertions.PlanMatchPattern.enforceSingleRow) Test(org.testng.annotations.Test) PARTITIONED(io.prestosql.spi.plan.JoinNode.DistributionType.PARTITIONED) PlanBuilder.castToRowExpression(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.castToRowExpression) PlanMatchPattern.values(io.prestosql.sql.planner.assertions.PlanMatchPattern.values) ImmutableList(com.google.common.collect.ImmutableList) RuleAssert(io.prestosql.sql.planner.iterative.rule.test.RuleAssert) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) Type(io.prestosql.spi.plan.JoinNode.Type) JoinNode(io.prestosql.spi.plan.JoinNode) TaskCountEstimator(io.prestosql.cost.TaskCountEstimator) Symbol(io.prestosql.spi.plan.Symbol) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) AfterClass(org.testng.annotations.AfterClass) PlanMatchPattern.join(io.prestosql.sql.planner.assertions.PlanMatchPattern.join) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) ImmutableMap(com.google.common.collect.ImmutableMap) FULL(io.prestosql.spi.plan.JoinNode.Type.FULL) BeforeClass(org.testng.annotations.BeforeClass) CostComparator(io.prestosql.cost.CostComparator) RIGHT(io.prestosql.spi.plan.JoinNode.Type.RIGHT) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) DistributionType(io.prestosql.spi.plan.JoinNode.DistributionType) INNER(io.prestosql.spi.plan.JoinNode.Type.INNER) Optional(java.util.Optional) JOIN_MAX_BROADCAST_TABLE_SIZE(io.prestosql.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE) RuleTester(io.prestosql.sql.planner.iterative.rule.test.RuleTester) VarcharType(io.prestosql.spi.type.VarcharType) JOIN_DISTRIBUTION_TYPE(io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) LEFT(io.prestosql.spi.plan.JoinNode.Type.LEFT) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Symbol(io.prestosql.spi.plan.Symbol) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) Test(org.testng.annotations.Test)

Example 3 with CostComparator

use of io.prestosql.cost.CostComparator in project hetu-core by openlookeng.

the class TestDetermineJoinDistributionType method testFlipAndReplicateRightOuterJoinWhenJoinCardinalityUnknown.

@Test
public void testFlipAndReplicateRightOuterJoinWhenJoinCardinalityUnknown() {
    int aRows = 10;
    int bRows = 1_000_000;
    assertDetermineJoinDistributionType(new CostComparator(75, 10, 15)).setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).overrideStats("valuesA", PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), SymbolStatsEstimate.unknown())).build()).overrideStats("valuesB", PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), SymbolStatsEstimate.unknown())).build()).on(p -> p.join(RIGHT, p.values(new PlanNodeId("valuesA"), aRows, p.symbol("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.symbol("B1", BIGINT)), ImmutableList.of(new JoinNode.EquiJoinClause(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT))), ImmutableList.of(p.symbol("A1", BIGINT), p.symbol("B1", BIGINT)), Optional.empty())).matches(join(LEFT, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}
Also used : CostComparator(io.prestosql.cost.CostComparator) PlanBuilder.constantExpressions(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.constantExpressions) SymbolStatsEstimate(io.prestosql.cost.SymbolStatsEstimate) REPLICATED(io.prestosql.spi.plan.JoinNode.DistributionType.REPLICATED) PlanMatchPattern.equiJoinClause(io.prestosql.sql.planner.assertions.PlanMatchPattern.equiJoinClause) JoinDistributionType(io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType) PlanMatchPattern.enforceSingleRow(io.prestosql.sql.planner.assertions.PlanMatchPattern.enforceSingleRow) Test(org.testng.annotations.Test) PARTITIONED(io.prestosql.spi.plan.JoinNode.DistributionType.PARTITIONED) PlanBuilder.castToRowExpression(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder.castToRowExpression) PlanMatchPattern.values(io.prestosql.sql.planner.assertions.PlanMatchPattern.values) ImmutableList(com.google.common.collect.ImmutableList) RuleAssert(io.prestosql.sql.planner.iterative.rule.test.RuleAssert) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) Type(io.prestosql.spi.plan.JoinNode.Type) JoinNode(io.prestosql.spi.plan.JoinNode) TaskCountEstimator(io.prestosql.cost.TaskCountEstimator) Symbol(io.prestosql.spi.plan.Symbol) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) AfterClass(org.testng.annotations.AfterClass) PlanMatchPattern.join(io.prestosql.sql.planner.assertions.PlanMatchPattern.join) PlanNodeStatsEstimate(io.prestosql.cost.PlanNodeStatsEstimate) ImmutableMap(com.google.common.collect.ImmutableMap) FULL(io.prestosql.spi.plan.JoinNode.Type.FULL) BeforeClass(org.testng.annotations.BeforeClass) CostComparator(io.prestosql.cost.CostComparator) RIGHT(io.prestosql.spi.plan.JoinNode.Type.RIGHT) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) DistributionType(io.prestosql.spi.plan.JoinNode.DistributionType) INNER(io.prestosql.spi.plan.JoinNode.Type.INNER) Optional(java.util.Optional) JOIN_MAX_BROADCAST_TABLE_SIZE(io.prestosql.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE) RuleTester(io.prestosql.sql.planner.iterative.rule.test.RuleTester) VarcharType(io.prestosql.spi.type.VarcharType) JOIN_DISTRIBUTION_TYPE(io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) LEFT(io.prestosql.spi.plan.JoinNode.Type.LEFT) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Symbol(io.prestosql.spi.plan.Symbol) Test(org.testng.annotations.Test)

Example 4 with CostComparator

use of io.prestosql.cost.CostComparator in project hetu-core by openlookeng.

the class TestJoinEnumerator method testDoesNotCreateJoinWhenPartitionedOnCrossJoin.

@Test
public void testDoesNotCreateJoinWhenPartitionedOnCrossJoin() {
    PlanNodeIdAllocator idAllocator = new PlanNodeIdAllocator();
    PlanBuilder p = new PlanBuilder(idAllocator, queryRunner.getMetadata());
    Symbol a1 = p.symbol("A1");
    Symbol b1 = p.symbol("B1");
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(queryRunner.getMetadata()), new FunctionResolution(queryRunner.getMetadata().getFunctionAndTypeManager()), queryRunner.getMetadata().getFunctionAndTypeManager());
    MultiJoinNode multiJoinNode = new MultiJoinNode(new LinkedHashSet<>(ImmutableList.of(p.values(a1), p.values(b1))), TRUE_CONSTANT, ImmutableList.of(a1, b1), logicalRowExpressions);
    JoinEnumerator joinEnumerator = new JoinEnumerator(new CostComparator(1, 1, 1), multiJoinNode.getFilter(), createContext(), queryRunner.getMetadata(), logicalRowExpressions);
    JoinEnumerationResult actual = joinEnumerator.createJoinAccordingToPartitioning(multiJoinNode.getSources(), multiJoinNode.getOutputSymbols(), ImmutableSet.of(0));
    assertFalse(actual.getPlanNode().isPresent());
    assertEquals(actual.getCost(), PlanCostEstimate.infinite());
}
Also used : CostComparator(io.prestosql.cost.CostComparator) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) JoinEnumerator(io.prestosql.sql.planner.iterative.rule.ReorderJoins.JoinEnumerator) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) Symbol(io.prestosql.spi.plan.Symbol) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) JoinEnumerationResult(io.prestosql.sql.planner.iterative.rule.ReorderJoins.JoinEnumerationResult) PlanBuilder(io.prestosql.sql.planner.iterative.rule.test.PlanBuilder) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) MultiJoinNode(io.prestosql.sql.planner.iterative.rule.ReorderJoins.MultiJoinNode) Test(org.testng.annotations.Test)

Example 5 with CostComparator

use of io.prestosql.cost.CostComparator in project hetu-core by openlookeng.

the class AbstractTestQueryFramework method getQueryExplainer.

private QueryExplainer getQueryExplainer() {
    Metadata metadata = queryRunner.getMetadata();
    FeaturesConfig featuresConfig = new FeaturesConfig().setOptimizeHashGeneration(true);
    boolean forceSingleNode = queryRunner.getNodeCount() == 1;
    TaskCountEstimator taskCountEstimator = new TaskCountEstimator(queryRunner::getNodeCount);
    CostCalculator costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator);
    HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
    List<PlanOptimizer> optimizers = new PlanOptimizers(metadata, new TypeAnalyzer(sqlParser, metadata), featuresConfig, new TaskManagerConfig(), forceSingleNode, new MBeanExporter(new TestingMBeanServer()), queryRunner.getSplitManager(), queryRunner.getPlanOptimizerManager(), queryRunner.getPageSourceManager(), queryRunner.getStatsCalculator(), costCalculator, new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator), new CostComparator(featuresConfig), taskCountEstimator, new CubeManager(featuresConfig, hetuMetaStoreManager)).get();
    return new QueryExplainer(optimizers, new PlanFragmenter(metadata, queryRunner.getNodePartitioningManager(), new QueryManagerConfig()), metadata, queryRunner.getAccessControl(), sqlParser, queryRunner.getStatsCalculator(), costCalculator, ImmutableMap.of(), new HeuristicIndexerManager(null, null), new CubeManager(featuresConfig, hetuMetaStoreManager));
}
Also used : TaskCountEstimator(io.prestosql.cost.TaskCountEstimator) CostComparator(io.prestosql.cost.CostComparator) PlanOptimizer(io.prestosql.sql.planner.optimizations.PlanOptimizer) TestingMBeanServer(org.weakref.jmx.testing.TestingMBeanServer) QueryExplainer(io.prestosql.sql.analyzer.QueryExplainer) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) MBeanExporter(org.weakref.jmx.MBeanExporter) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) Metadata(io.prestosql.metadata.Metadata) PlanFragmenter(io.prestosql.sql.planner.PlanFragmenter) TaskManagerConfig(io.prestosql.execution.TaskManagerConfig) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) CubeManager(io.prestosql.cube.CubeManager) CostCalculator(io.prestosql.cost.CostCalculator) PlanOptimizers(io.prestosql.sql.planner.PlanOptimizers) QueryManagerConfig(io.prestosql.execution.QueryManagerConfig) CostCalculatorUsingExchanges(io.prestosql.cost.CostCalculatorUsingExchanges) CostCalculatorWithEstimatedExchanges(io.prestosql.cost.CostCalculatorWithEstimatedExchanges) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager)

Aggregations

CostComparator (io.prestosql.cost.CostComparator)5 TaskCountEstimator (io.prestosql.cost.TaskCountEstimator)4 Symbol (io.prestosql.spi.plan.Symbol)4 Test (org.testng.annotations.Test)4 ImmutableList (com.google.common.collect.ImmutableList)3 ImmutableMap (com.google.common.collect.ImmutableMap)3 JOIN_DISTRIBUTION_TYPE (io.prestosql.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE)3 JOIN_MAX_BROADCAST_TABLE_SIZE (io.prestosql.SystemSessionProperties.JOIN_MAX_BROADCAST_TABLE_SIZE)3 PlanNodeStatsEstimate (io.prestosql.cost.PlanNodeStatsEstimate)3 SymbolStatsEstimate (io.prestosql.cost.SymbolStatsEstimate)3 JoinNode (io.prestosql.spi.plan.JoinNode)3 DistributionType (io.prestosql.spi.plan.JoinNode.DistributionType)3 PARTITIONED (io.prestosql.spi.plan.JoinNode.DistributionType.PARTITIONED)3 REPLICATED (io.prestosql.spi.plan.JoinNode.DistributionType.REPLICATED)3 Type (io.prestosql.spi.plan.JoinNode.Type)3 FULL (io.prestosql.spi.plan.JoinNode.Type.FULL)3 INNER (io.prestosql.spi.plan.JoinNode.Type.INNER)3 LEFT (io.prestosql.spi.plan.JoinNode.Type.LEFT)3 RIGHT (io.prestosql.spi.plan.JoinNode.Type.RIGHT)3 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)3