Examples with TableScanNode io.prestosql.spi.plan.TableScanNode used on opensource projects

Example 26 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class TestCostCalculator method testUnion.

@Test
public void testUnion() {
    TableScanNode ts1 = tableScan("ts1", "orderkey");
    TableScanNode ts2 = tableScan("ts2", "orderkey_0");
    ImmutableListMultimap.Builder<Symbol, Symbol> outputMappings = ImmutableListMultimap.builder();
    outputMappings.put(new Symbol("orderkey_1"), new Symbol("orderkey"));
    outputMappings.put(new Symbol("orderkey_1"), new Symbol("orderkey_0"));
    UnionNode union = new UnionNode(new PlanNodeId("union"), ImmutableList.of(ts1, ts2), outputMappings.build(), ImmutableList.of(new Symbol("orderkey_1")));
    Map<String, PlanNodeStatsEstimate> stats = ImmutableMap.of("ts1", statsEstimate(ts1, 4000), "ts2", statsEstimate(ts2, 1000), "union", statsEstimate(ts1, 5000));
    Map<String, PlanCostEstimate> costs = ImmutableMap.of("ts1", cpuCost(1000), "ts2", cpuCost(1000));
    Map<String, Type> types = ImmutableMap.of("orderkey", BIGINT, "orderkey_0", BIGINT, "orderkey_1", BIGINT);
    assertCost(union, costs, stats, types).cpu(2000).memory(0).network(0);
    assertCostEstimatedExchanges(union, costs, stats, types).cpu(2000).memory(0).network(5000 * IS_NULL_OVERHEAD);
}

Example 27 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class TestCostCalculator method testRepartitionedJoinWithExchange.

@Test
public void testRepartitionedJoinWithExchange() {
    TableScanNode ts1 = tableScan("ts1", "orderkey");
    TableScanNode ts2 = tableScan("ts2", "orderkey_0");
    ExchangeNode remoteExchange1 = partitionedExchange(new PlanNodeId("re1"), REMOTE, ts1, ImmutableList.of(new Symbol("orderkey")), Optional.empty());
    ExchangeNode remoteExchange2 = partitionedExchange(new PlanNodeId("re2"), REMOTE, ts2, ImmutableList.of(new Symbol("orderkey_0")), Optional.empty());
    ExchangeNode localExchange = partitionedExchange(new PlanNodeId("le"), LOCAL, remoteExchange2, ImmutableList.of(new Symbol("orderkey_0")), Optional.empty());
    JoinNode join = join("join", remoteExchange1, localExchange, JoinNode.DistributionType.PARTITIONED, "orderkey", "orderkey_0");
    Map<String, PlanNodeStatsEstimate> stats = ImmutableMap.<String, PlanNodeStatsEstimate>builder().put("join", statsEstimate(join, 12000)).put("re1", statsEstimate(remoteExchange1, 10000)).put("re2", statsEstimate(remoteExchange2, 10000)).put("le", statsEstimate(localExchange, 6000)).put("ts1", statsEstimate(ts1, 6000)).put("ts2", statsEstimate(ts2, 1000)).build();
    Map<String, Type> types = ImmutableMap.of("orderkey", BIGINT, "orderkey_0", BIGINT);
    assertFragmentedEqualsUnfragmented(join, stats, types);
}

Example 28 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class TestCostCalculator method testFilter.

@Test
public void testFilter() {
    TableScanNode tableScan = tableScan("ts", "string");
    IsNullPredicate expression = new IsNullPredicate(new SymbolReference("string"));
    FilterNode filter = new FilterNode(new PlanNodeId("filter"), tableScan, castToRowExpression(expression));
    Map<String, PlanCostEstimate> costs = ImmutableMap.of("ts", cpuCost(1000));
    Map<String, PlanNodeStatsEstimate> stats = ImmutableMap.of("filter", statsEstimate(filter, 4000), "ts", statsEstimate(tableScan, 1000));
    Map<String, Type> types = ImmutableMap.of("string", VARCHAR);
    assertCost(filter, costs, stats, types).cpu(1000 + 1000 * OFFSET_AND_IS_NULL_OVERHEAD).memory(0).network(0);
    assertCostEstimatedExchanges(filter, costs, stats, types).cpu(1000 + 1000 * OFFSET_AND_IS_NULL_OVERHEAD).memory(0).network(0);
    assertCostFragmentedPlan(filter, costs, stats, types).cpu(1000 + 1000 * OFFSET_AND_IS_NULL_OVERHEAD).memory(0).network(0);
    assertCostHasUnknownComponentsForUnknownStats(filter, types);
}

Example 29 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class PushPredicateIntoTableScan method pushPredicateIntoTableScan.

/**
 * For RowExpression {@param predicate}
 */
public static Optional<PlanNode> pushPredicateIntoTableScan(TableScanNode node, RowExpression predicate, boolean pruneWithPredicateExpression, Session session, PlanNodeIdAllocator idAllocator, PlanSymbolAllocator planSymbolAllocator, Metadata metadata, RowExpressionDomainTranslator domainTranslator, boolean pushPartitionsOnly) {
    // don't include non-deterministic predicates
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(metadata), new FunctionResolution(metadata.getFunctionAndTypeManager()), metadata.getFunctionAndTypeManager());
    RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(predicate);
    RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression> decomposedPredicate = domainTranslator.fromPredicate(session.toConnectorSession(), deterministicPredicate);
    TupleDomain<ColumnHandle> newDomain = decomposedPredicate.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName()))).intersect(node.getEnforcedConstraint());
    Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
    Set<ColumnHandle> allColumnHandles = new HashSet<>();
    assignments.keySet().stream().forEach(allColumnHandles::add);
    Constraint constraint;
    List<Constraint> disjunctConstraints = ImmutableList.of();
    if (!pushPartitionsOnly) {
        List<RowExpression> orSet = LogicalRowExpressions.extractDisjuncts(decomposedPredicate.getRemainingExpression());
        List<RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression>> disjunctPredicates = orSet.stream().map(e -> domainTranslator.fromPredicate(session.toConnectorSession(), e)).collect(Collectors.toList());
        /* Check if any Branch yeild all records; then no need to process OR branches */
        if (!disjunctPredicates.stream().anyMatch(e -> e.getTupleDomain().isAll())) {
            List<TupleDomain<ColumnHandle>> orDomains = disjunctPredicates.stream().map(er -> er.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName())))).collect(Collectors.toList());
            disjunctConstraints = orDomains.stream().filter(d -> !d.isAll() && !d.isNone()).map(d -> new Constraint(d)).collect(Collectors.toList());
        }
    }
    if (pruneWithPredicateExpression) {
        LayoutConstraintEvaluatorForRowExpression evaluator = new LayoutConstraintEvaluatorForRowExpression(metadata, session, node.getAssignments(), logicalRowExpressions.combineConjuncts(deterministicPredicate, // which would be expensive to evaluate in the call to isCandidate below.
        domainTranslator.toPredicate(newDomain.simplify().transform(column -> {
            if (assignments.size() == 0 || assignments.getOrDefault(column, null) == null) {
                return null;
            } else {
                return new VariableReferenceExpression(assignments.getOrDefault(column, null).getName(), planSymbolAllocator.getSymbols().get(assignments.getOrDefault(column, null)));
            }
        }))));
        constraint = new Constraint(newDomain, evaluator::isCandidate);
    } else {
        // Currently, invoking the expression interpreter is very expensive.
        // TODO invoke the interpreter unconditionally when the interpreter becomes cheap enough.
        constraint = new Constraint(newDomain);
    }
    TableHandle newTable;
    TupleDomain<ColumnHandle> remainingFilter;
    if (!metadata.usesLegacyTableLayouts(session, node.getTable())) {
        if (newDomain.isNone()) {
            // to turn the subtree into a Values node
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, node.getTable(), constraint, disjunctConstraints, allColumnHandles, pushPartitionsOnly);
        if (!result.isPresent()) {
            return Optional.empty();
        }
        newTable = result.get().getHandle();
        if (metadata.getTableProperties(session, newTable).getPredicate().isNone()) {
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        remainingFilter = result.get().getRemainingFilter();
    } else {
        Optional<TableLayoutResult> layout = metadata.getLayout(session, node.getTable(), constraint, Optional.of(node.getOutputSymbols().stream().map(node.getAssignments()::get).collect(toImmutableSet())));
        if (!layout.isPresent() || layout.get().getTableProperties().getPredicate().isNone()) {
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        newTable = layout.get().getNewTableHandle();
        remainingFilter = layout.get().getUnenforcedConstraint();
    }
    TableScanNode tableScan = new TableScanNode(node.getId(), newTable, node.getOutputSymbols(), node.getAssignments(), computeEnforced(newDomain, remainingFilter), Optional.of(deterministicPredicate), node.getStrategy(), node.getReuseTableScanMappingId(), 0, node.isForDelete());
    // The order of the arguments to combineConjuncts matters:
    // * Unenforced constraints go first because they can only be simple column references,
    // which are not prone to logic errors such as out-of-bound access, div-by-zero, etc.
    // * Conjuncts in non-deterministic expressions and non-TupleDomain-expressible expressions should
    // retain their original (maybe intermixed) order from the input predicate. However, this is not implemented yet.
    // * Short of implementing the previous bullet point, the current order of non-deterministic expressions
    // and non-TupleDomain-expressible expressions should be retained. Changing the order can lead
    // to failures of previously successful queries.
    RowExpression resultingPredicate;
    if (remainingFilter.isAll() && newTable.getConnectorHandle().hasDisjunctFiltersPushdown()) {
        resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate));
    } else {
        resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate), decomposedPredicate.getRemainingExpression());
    }
    if (!TRUE_CONSTANT.equals(resultingPredicate)) {
        return Optional.of(new FilterNode(idAllocator.getNextId(), tableScan, resultingPredicate));
    }
    return Optional.of(tableScan);
}

Example 30 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class PushProjectionIntoTableScan method apply.

@Override
public Result apply(ProjectNode project, Captures captures, Context context) {
    TableScanNode tableScan = captures.get(TABLE_SCAN);
    List<ConnectorExpression> projections;
    try {
        projections = project.getAssignments().getExpressions().stream().map(expression -> ConnectorExpressionTranslator.translate(expression)).collect(toImmutableList());
    } catch (UnsupportedOperationException e) {
        // rewritten in terms of the new assignments for the columns passed in #2
        return Result.empty();
    }
    Map<String, ColumnHandle> assignments = tableScan.getAssignments().entrySet().stream().collect(toImmutableMap(entry -> entry.getKey().getName(), Map.Entry::getValue));
    Optional<ProjectionApplicationResult<TableHandle>> result = metadata.applyProjection(context.getSession(), tableScan.getTable(), projections, assignments);
    if (!result.isPresent()) {
        return Result.empty();
    }
    List<Symbol> newScanOutputs = new ArrayList<>();
    Map<Symbol, ColumnHandle> newScanAssignments = new HashMap<>();
    Map<String, Symbol> variableMappings = new HashMap<>();
    for (ProjectionApplicationResult.Assignment assignment : result.get().getAssignments()) {
        Symbol symbol = context.getSymbolAllocator().newSymbol(assignment.getVariable(), assignment.getType());
        newScanOutputs.add(symbol);
        newScanAssignments.put(symbol, assignment.getColumn());
        variableMappings.put(assignment.getVariable(), symbol);
    }
    // TODO: ensure newProjections.size == original projections.size
    List<RowExpression> newProjections = result.get().getProjections().stream().map(expression -> ConnectorExpressionTranslator.translate(expression, variableMappings, new LiteralEncoder(metadata))).collect(toImmutableList());
    Assignments.Builder newProjectionAssignments = Assignments.builder();
    for (int i = 0; i < project.getOutputSymbols().size(); i++) {
        newProjectionAssignments.put(project.getOutputSymbols().get(i), newProjections.get(i));
    }
    return Result.ofPlanNode(new ProjectNode(context.getIdAllocator().getNextId(), TableScanNode.newInstance(tableScan.getId(), result.get().getHandle(), newScanOutputs, newScanAssignments, tableScan.getStrategy(), tableScan.getReuseTableScanMappingId(), tableScan.getConsumerTableScanNodeCount(), tableScan.isForDelete()), newProjectionAssignments.build()));
}