Examples with TableScanNode io.prestosql.spi.plan.TableScanNode used on opensource projects

Example 16 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class SimpleNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks, Optional<SqlStageExecution> stage) {
    Multimap<InternalNode, Split> assignment = HashMultimap.create();
    NodeMap nodeMapSlice = this.nodeMap.get().get();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMapSlice, existingTasks);
    ResettableRandomizedIterator<InternalNode> randomCandidates = randomizedNodes(nodeMapSlice, ImmutableSet.of());
    Set<InternalNode> blockedExactNodes = new HashSet<>();
    boolean splitWaitingForAnyNode = false;
    // splitsToBeRedistributed becomes true only when splits go through locality-based assignment
    boolean splitsToBeRedistributed = false;
    Set<Split> remainingSplits = new HashSet<>();
    // Check if the current stage has a TableScanNode which is reading the table for the 2nd time or beyond
    if (stage.isPresent() && stage.get().getStateMachine().getConsumerScanNode() != null) {
        try {
            // if node exists, get the TableScanNode and cast it as consumer
            TableScanNode consumer = stage.get().getStateMachine().getConsumerScanNode();
            // all tables part of this stage
            Map<PlanNodeId, TableInfo> tables = stage.get().getStageInfo().getTables();
            QualifiedObjectName tableName;
            for (Map.Entry<PlanNodeId, TableInfo> entry : tables.entrySet()) {
                tableName = entry.getValue().getTableName();
                if (tableSplitAssignmentInfo.getReuseTableScanMappingIdSplitAssignmentMap().containsKey(consumer.getReuseTableScanMappingId())) {
                    // compare splitkey using equals and then assign nodes accordingly.
                    HashMap<SplitKey, InternalNode> splitKeyNodeAssignment = tableSplitAssignmentInfo.getSplitKeyNodeAssignment(consumer.getReuseTableScanMappingId());
                    Set<SplitKey> splitKeySet = splitKeyNodeAssignment.keySet();
                    assignment.putAll(createConsumerScanNodeAssignment(tableName, splits, splitKeySet, splitKeyNodeAssignment));
                    for (Map.Entry<InternalNode, Split> nodeAssignmentEntry : assignment.entries()) {
                        InternalNode node = nodeAssignmentEntry.getKey();
                        assignmentStats.addAssignedSplit(node);
                    }
                }
            }
            log.debug("Consumer:: Assignment size is " + assignment.size() + " ,Assignment is " + assignment + " ,Assignment Stats is " + assignmentStats);
        } catch (NotImplementedException e) {
            log.error("Not a Hive Split! Other Connector Splits not supported currently. Error: " + e);
            throw new UnsupportedOperationException("Not a Hive Split! Other Connector Splits not supported currently. Error: " + e);
        }
    } else {
        // optimizedLocalScheduling enables prioritized assignment of splits to local nodes when splits contain locality information
        if (optimizedLocalScheduling) {
            // should not hit for consumer case
            for (Split split : splits) {
                if (split.isRemotelyAccessible() && !split.getAddresses().isEmpty()) {
                    List<InternalNode> candidateNodes = selectExactNodes(nodeMapSlice, split.getAddresses(), includeCoordinator);
                    Optional<InternalNode> chosenNode = candidateNodes.stream().filter(ownerNode -> assignmentStats.getTotalSplitCount(ownerNode) < maxSplitsPerNode).min(comparingInt(assignmentStats::getTotalSplitCount));
                    if (chosenNode.isPresent()) {
                        assignment.put(chosenNode.get(), split);
                        // check later
                        assignmentStats.addAssignedSplit(chosenNode.get());
                        splitsToBeRedistributed = true;
                        continue;
                    }
                }
                remainingSplits.add(split);
            }
        } else {
            remainingSplits = splits;
        }
        for (Split split : remainingSplits) {
            randomCandidates.reset();
            List<InternalNode> candidateNodes;
            if (!split.isRemotelyAccessible()) {
                candidateNodes = selectExactNodes(nodeMapSlice, split.getAddresses(), includeCoordinator);
            } else {
                candidateNodes = selectNodes(minCandidates, randomCandidates);
            }
            if (candidateNodes.isEmpty()) {
                log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMapSlice.getNodesByHost().keys());
                throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query");
            }
            InternalNode chosenNode = null;
            int min = Integer.MAX_VALUE;
            for (InternalNode node : candidateNodes) {
                int totalSplitCount = assignmentStats.getTotalSplitCount(node);
                if (totalSplitCount < min && totalSplitCount < maxSplitsPerNode) {
                    chosenNode = node;
                    min = totalSplitCount;
                }
            }
            if (chosenNode == null) {
                // min is guaranteed to be MAX_VALUE at this line
                for (InternalNode node : candidateNodes) {
                    int totalSplitCount = assignmentStats.getQueuedSplitCountForStage(node);
                    if (totalSplitCount < min && totalSplitCount < maxPendingSplitsPerTask) {
                        chosenNode = node;
                        min = totalSplitCount;
                    }
                }
            }
            if (chosenNode != null) {
                assignment.put(chosenNode, split);
                assignmentStats.addAssignedSplit(chosenNode);
            } else {
                if (split.isRemotelyAccessible()) {
                    splitWaitingForAnyNode = true;
                } else // Exact node set won't matter, if a split is waiting for any node
                if (!splitWaitingForAnyNode) {
                    blockedExactNodes.addAll(candidateNodes);
                }
            }
        }
    }
    ListenableFuture<?> blocked;
    if (splitWaitingForAnyNode) {
        blocked = toWhenHasSplitQueueSpaceFuture(existingTasks, calculateLowWatermark(maxPendingSplitsPerTask));
    } else {
        blocked = toWhenHasSplitQueueSpaceFuture(blockedExactNodes, existingTasks, calculateLowWatermark(maxPendingSplitsPerTask));
    }
    if (!stage.isPresent() || stage.get().getStateMachine().getConsumerScanNode() == null) {
        if (splitsToBeRedistributed) {
            // skip for consumer
            equateDistribution(assignment, assignmentStats, nodeMapSlice);
        }
    }
    // Check if the current stage has a TableScanNode which is reading the table for the 1st time
    if (stage.isPresent() && stage.get().getStateMachine().getProducerScanNode() != null) {
        // if node exists, get the TableScanNode and annotate it as producer
        saveProducerScanNodeAssignment(stage, assignment, assignmentStats);
    }
    // Check if its CTE node and its feeder
    if (stage.isPresent() && stage.get().getFragment().getFeederCTEId().isPresent()) {
        updateFeederNodeAndSplitCount(stage.get(), assignment);
    }
    return new SplitPlacementResult(blocked, assignment);
}

Example 17 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class SplitFiltering method isSplitFilterApplicable.

public static boolean isSplitFilterApplicable(SqlStageExecution stage) {
    List<PlanNode> filterNodeOptional = getFilterNode(stage);
    if (filterNodeOptional.isEmpty()) {
        return false;
    }
    PlanNode node = filterNodeOptional.get(0);
    if (node instanceof FilterNode) {
        FilterNode filterNode = (FilterNode) node;
        PlanNode sourceNode = filterNode.getSource();
        if (!(sourceNode instanceof TableScanNode)) {
            return false;
        }
        // if a catalog name starts with a $, it's not an normal query, could be something like show tables;
        TableHandle table = ((TableScanNode) sourceNode).getTable();
        String catalogName = table.getCatalogName().getCatalogName();
        if (catalogName.startsWith("$")) {
            return false;
        }
        if (!table.getConnectorHandle().isFilterSupported()) {
            return false;
        }
        if (!isSupportedExpression(filterNode.getPredicate()) && (!((TableScanNode) sourceNode).getPredicate().isPresent() || !isSupportedExpression(((TableScanNode) sourceNode).getPredicate().get()))) {
            return false;
        }
    }
    if (node instanceof TableScanNode) {
        TableScanNode tableScanNode = (TableScanNode) node;
        // if a catalog name starts with a $, it's not an normal query, could be something like show tables;
        TableHandle table = tableScanNode.getTable();
        String catalogName = table.getCatalogName().getCatalogName();
        if (catalogName.startsWith("$")) {
            return false;
        }
        if (!table.getConnectorHandle().isFilterSupported()) {
            return false;
        }
        if (!tableScanNode.getPredicate().isPresent() || !isSupportedExpression(tableScanNode.getPredicate().get())) {
            return false;
        }
    }
    return true;
}

Example 18 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class SplitFiltering method getFullyQualifiedName.

public static Optional<String> getFullyQualifiedName(SqlStageExecution stage) {
    List<PlanNode> filterNodeOptional = getFilterNode(stage);
    if (filterNodeOptional.size() == 0) {
        return Optional.empty();
    }
    TableScanNode tableScanNode;
    if (filterNodeOptional.get(0) instanceof FilterNode) {
        FilterNode filterNode = (FilterNode) filterNodeOptional.get(0);
        tableScanNode = (TableScanNode) filterNode.getSource();
    } else {
        tableScanNode = (TableScanNode) filterNodeOptional.get(0);
    }
    String fullQualifiedTableName = tableScanNode.getTable().getFullyQualifiedName();
    return Optional.of(fullQualifiedTableName);
}

Example 19 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class SplitFiltering method getFilterNode.

private static List<PlanNode> getFilterNode(SqlStageExecution stage) {
    PlanFragment fragment = stage.getFragment();
    PlanNode root = fragment.getRoot();
    List<PlanNode> result = new LinkedList<>();
    Queue<PlanNode> queue = new LinkedList<>();
    queue.add(root);
    while (!queue.isEmpty()) {
        PlanNode node = queue.poll();
        if (node instanceof FilterNode || node instanceof TableScanNode) {
            result.add(node);
        }
        queue.addAll(node.getSources());
    }
    return result;
}

Example 20 with TableScanNode

use of io.prestosql.spi.plan.TableScanNode in project hetu-core by openlookeng.

the class TestCubeStatementGenerator method setup.

@BeforeClass
public void setup() {
    planBuilder = new PlanBuilder(new PlanNodeIdAllocator(), dummyMetadata());
    symbolAllocator = new PlanSymbolAllocator();
    builder = CubeStatement.newBuilder();
    columnOrderkey = symbolAllocator.newSymbol("orderkey", BIGINT);
    columnTotalprice = symbolAllocator.newSymbol("totalprice", DOUBLE);
    columnAvgPrice = symbolAllocator.newSymbol("avgprice", DOUBLE);
    orderkeyHandle = new TpchColumnHandle("orderkey", BIGINT);
    totalpriceHandle = new TpchColumnHandle("totalprice", DOUBLE);
    columnMapping = new HashMap<>();
    columnMapping.put("orderkey", orderkeyHandle);
    columnMapping.put("totalprice", totalpriceHandle);
    columnMapping.put("avgprice", columnAvgPrice);
    Map<Symbol, ColumnHandle> assignments = ImmutableMap.<Symbol, ColumnHandle>builder().put(columnOrderkey, orderkeyHandle).put(columnTotalprice, totalpriceHandle).build();
    TpchTableHandle orders = new TpchTableHandle("orders", 1.0);
    TableHandle ordersTableHandle = new TableHandle(new CatalogName("test"), orders, TpchTransactionHandle.INSTANCE, Optional.of(new TpchTableLayoutHandle(orders, TupleDomain.all())));
    baseTableScan = new TableScanNode(new PlanNodeId(UUID.randomUUID().toString()), ordersTableHandle, ImmutableList.copyOf(assignments.keySet()), assignments, Optional.empty(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, false);
}