Examples with PlanNodeStatsEstimate io.prestosql.cost.PlanNodeStatsEstimate used on opensource projects

Example 6 with PlanNodeStatsEstimate

use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.

the class TestDetermineJoinDistributionType method testReplicatesWhenNotRestricted.

@Test
public void testReplicatesWhenNotRestricted() {
    // variable width so that average row size is respected
    VarcharType symbolType = createUnboundedVarcharType();
    int aRows = 10_000;
    int bRows = 10;
    PlanNodeStatsEstimate probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
    PlanNodeStatsEstimate buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 10))).build();
    // B table is small enough to be replicated in AUTOMATIC_RESTRICTED mode
    assertDetermineJoinDistributionType().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).on(p -> {
        Symbol a1 = p.symbol("A1", symbolType);
        Symbol b1 = p.symbol("B1", symbolType);
        return p.join(INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty());
    }).matches(join(INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
    probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
    buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
    // B table exceeds AUTOMATIC_RESTRICTED limit therefore it is partitioned
    assertDetermineJoinDistributionType().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).on(p -> {
        Symbol a1 = p.symbol("A1", symbolType);
        Symbol b1 = p.symbol("B1", symbolType);
        return p.join(INNER, p.values(new PlanNodeId("valuesA"), aRows, a1), p.values(new PlanNodeId("valuesB"), bRows, b1), ImmutableList.of(new JoinNode.EquiJoinClause(a1, b1)), ImmutableList.of(a1, b1), Optional.empty());
    }).matches(join(INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(PARTITIONED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}

Example 7 with PlanNodeStatsEstimate

use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.

the class DetermineJoinDistributionType method canReplicate.

public static boolean canReplicate(JoinNode joinNode, Context context) {
    JoinDistributionType joinDistributionType = getJoinDistributionType(context.getSession());
    if (!joinDistributionType.canReplicate()) {
        return false;
    }
    Optional<DataSize> joinMaxBroadcastTableSize = getJoinMaxBroadcastTableSize(context.getSession());
    if (!joinMaxBroadcastTableSize.isPresent()) {
        return true;
    }
    PlanNode buildSide = joinNode.getRight();
    PlanNodeStatsEstimate buildSideStatsEstimate = context.getStatsProvider().getStats(buildSide);
    double buildSideSizeInBytes = buildSideStatsEstimate.getOutputSizeInBytes(buildSide.getOutputSymbols(), context.getSymbolAllocator().getTypes());
    return buildSideSizeInBytes <= joinMaxBroadcastTableSize.get().toBytes() || getSourceTablesSizeInBytes(buildSide, context) <= joinMaxBroadcastTableSize.get().toBytes();
}

Example 8 with PlanNodeStatsEstimate

use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.

the class DetermineSemiJoinDistributionType method canReplicate.

private boolean canReplicate(SemiJoinNode node, Context context) {
    Optional<DataSize> joinMaxBroadcastTableSize = getJoinMaxBroadcastTableSize(context.getSession());
    if (!joinMaxBroadcastTableSize.isPresent()) {
        return true;
    }
    PlanNode buildSide = node.getFilteringSource();
    PlanNodeStatsEstimate buildSideStatsEstimate = context.getStatsProvider().getStats(buildSide);
    double buildSideSizeInBytes = buildSideStatsEstimate.getOutputSizeInBytes(buildSide.getOutputSymbols(), context.getSymbolAllocator().getTypes());
    return buildSideSizeInBytes <= joinMaxBroadcastTableSize.get().toBytes() || getSourceTablesSizeInBytes(buildSide, context) <= joinMaxBroadcastTableSize.get().toBytes();
}

Example 9 with PlanNodeStatsEstimate

use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.

the class TestReorderJoins method testReplicatedScalarJoinEvenWhereSessionRequiresRepartitioned.

@Test
public void testReplicatedScalarJoinEvenWhereSessionRequiresRepartitioned() {
    PlanMatchPattern expectedPlan = join(INNER, ImmutableList.of(equiJoinClause("A1", "B1")), Optional.empty(), Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0)));
    PlanNodeStatsEstimate valuesA = PlanNodeStatsEstimate.builder().setOutputRowCount(10000).addSymbolStatistics(ImmutableMap.of(new Symbol("A1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build();
    PlanNodeStatsEstimate valuesB = PlanNodeStatsEstimate.builder().setOutputRowCount(10000).addSymbolStatistics(ImmutableMap.of(new Symbol("B1"), new SymbolStatsEstimate(0, 100, 0, 640000, 100))).build();
    assertReorderJoins().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.PARTITIONED.name()).on(p -> p.join(INNER, // matches isAtMostScalar
    p.values(new PlanNodeId("valuesA"), p.symbol("A1")), p.values(new PlanNodeId("valuesB"), ImmutableList.of(p.symbol("B1")), TWO_ROWS), ImmutableList.of(new EquiJoinClause(p.symbol("A1"), p.symbol("B1"))), ImmutableList.of(p.symbol("A1"), p.symbol("B1")), Optional.empty())).overrideStats("valuesA", valuesA).overrideStats("valuesB", valuesB).matches(expectedPlan);
    assertReorderJoins().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.PARTITIONED.name()).on(p -> p.join(INNER, p.values(new PlanNodeId("valuesB"), ImmutableList.of(p.symbol("B1")), TWO_ROWS), // matches isAtMostScalar
    p.values(new PlanNodeId("valuesA"), p.symbol("A1")), ImmutableList.of(new EquiJoinClause(p.symbol("A1"), p.symbol("B1"))), ImmutableList.of(p.symbol("A1"), p.symbol("B1")), Optional.empty())).overrideStats("valuesA", valuesA).overrideStats("valuesB", valuesB).matches(expectedPlan);
}

Example 10 with PlanNodeStatsEstimate

use of io.prestosql.cost.PlanNodeStatsEstimate in project hetu-core by openlookeng.

the class MetricComparator method getEstimatedValuesInternal.

private static List<OptionalDouble> getEstimatedValuesInternal(List<Metric> metrics, String query, QueryRunner runner, Session session) // TODO inline back this method
{
    Plan queryPlan = runner.createPlan(session, query, WarningCollector.NOOP);
    OutputNode outputNode = (OutputNode) queryPlan.getRoot();
    PlanNodeStatsEstimate outputNodeStats = queryPlan.getStatsAndCosts().getStats().getOrDefault(queryPlan.getRoot().getId(), PlanNodeStatsEstimate.unknown());
    StatsContext statsContext = buildStatsContext(queryPlan, outputNode);
    return getEstimatedValues(metrics, outputNodeStats, statsContext);
}