Examples with SplitContext com.facebook.presto.spi.SplitContext used on opensource projects

Example 6 with SplitContext

use of com.facebook.presto.spi.SplitContext in project presto by prestodb.

the class PageSourceManager method createPageSource.

@Override
public ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List<ColumnHandle> columns) {
    requireNonNull(split, "split is null");
    requireNonNull(columns, "columns is null");
    Optional<Supplier<TupleDomain<ColumnHandle>>> dynamicFilter = table.getDynamicFilter();
    // directly return the result if the given constraint is always false
    if (dynamicFilter.isPresent() && dynamicFilter.get().get().isNone()) {
        return new FixedPageSource(ImmutableList.of());
    }
    if (dynamicFilter.isPresent()) {
        split = new Split(split.getConnectorId(), split.getTransactionHandle(), split.getConnectorSplit(), split.getLifespan(), new SplitContext(split.getSplitContext().isCacheable(), dynamicFilter.get().get()));
    }
    ConnectorSession connectorSession = session.toConnectorSession(split.getConnectorId());
    if (table.getLayout().isPresent()) {
        return getPageSourceProvider(split).createPageSource(split.getTransactionHandle(), connectorSession, split.getConnectorSplit(), table.getLayout().get(), columns, split.getSplitContext());
    }
    return getPageSourceProvider(split).createPageSource(split.getTransactionHandle(), connectorSession, split.getConnectorSplit(), columns, split.getSplitContext());
}

Example 7 with SplitContext

use of com.facebook.presto.spi.SplitContext in project presto by prestodb.

the class IcebergPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit connectorSplit, ConnectorTableLayoutHandle layout, List<ColumnHandle> columns, SplitContext splitContext) {
    IcebergSplit split = (IcebergSplit) connectorSplit;
    IcebergTableLayoutHandle icebergLayout = (IcebergTableLayoutHandle) layout;
    IcebergTableHandle table = icebergLayout.getTable();
    List<IcebergColumnHandle> icebergColumns = columns.stream().map(IcebergColumnHandle.class::cast).collect(toImmutableList());
    Map<Integer, String> partitionKeys = split.getPartitionKeys();
    List<IcebergColumnHandle> regularColumns = columns.stream().map(IcebergColumnHandle.class::cast).filter(column -> !partitionKeys.containsKey(column.getId())).collect(toImmutableList());
    // TODO: pushdownFilter for icebergLayout
    HdfsContext hdfsContext = new HdfsContext(session, table.getSchemaName(), table.getTableName());
    ConnectorPageSource dataPageSource = createDataPageSource(session, hdfsContext, new Path(split.getPath()), split.getStart(), split.getLength(), split.getFileFormat(), table.getSchemaTableName(), regularColumns, table.getPredicate(), splitContext.isCacheable());
    return new IcebergPageSource(icebergColumns, partitionKeys, dataPageSource, session.getSqlFunctionProperties().getTimeZoneKey());
}

Example 8 with SplitContext

use of com.facebook.presto.spi.SplitContext in project presto by prestodb.

the class SimpleTtlNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks) {
    boolean isNodeSelectionStrategyNoPreference = splits.stream().allMatch(split -> split.getNodeSelectionStrategy() == NodeSelectionStrategy.NO_PREFERENCE);
    // Current NodeSelectionStrategy support is limited to NO_PREFERENCE
    if (!isNodeSelectionStrategyNoPreference) {
        return simpleNodeSelector.computeAssignments(splits, existingTasks);
    }
    ImmutableMultimap.Builder<InternalNode, Split> assignment = ImmutableMultimap.builder();
    NodeMap nodeMap = this.nodeMap.get().get();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks);
    List<InternalNode> eligibleNodes = getEligibleNodes(maxTasksPerStage, nodeMap, existingTasks);
    NodeSelection randomNodeSelection = new RandomNodeSelection(eligibleNodes, minCandidates);
    boolean splitWaitingForAnyNode = false;
    OptionalInt preferredNodeCount = OptionalInt.empty();
    for (Split split : splits) {
        if (split.getNodeSelectionStrategy() != NodeSelectionStrategy.NO_PREFERENCE) {
            throw new PrestoException(NODE_SELECTION_NOT_SUPPORTED, format("Unsupported node selection strategy for TTL scheduling: %s", split.getNodeSelectionStrategy()));
        }
        List<InternalNode> candidateNodes = randomNodeSelection.pickNodes(split);
        if (candidateNodes.isEmpty()) {
            log.warn("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getActiveNodes());
            throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query");
        }
        SplitWeight splitWeight = split.getSplitWeight();
        Optional<InternalNodeInfo> chosenNodeInfo = simpleNodeSelector.chooseLeastBusyNode(splitWeight, candidateNodes, assignmentStats::getTotalSplitsWeight, preferredNodeCount, maxSplitsWeightPerNode, assignmentStats);
        if (!chosenNodeInfo.isPresent()) {
            chosenNodeInfo = simpleNodeSelector.chooseLeastBusyNode(splitWeight, candidateNodes, assignmentStats::getQueuedSplitsWeightForStage, preferredNodeCount, maxPendingSplitsWeightPerTask, assignmentStats);
        }
        if (chosenNodeInfo.isPresent()) {
            split = new Split(split.getConnectorId(), split.getTransactionHandle(), split.getConnectorSplit(), split.getLifespan(), new SplitContext(chosenNodeInfo.get().isCacheable()));
            InternalNode chosenNode = chosenNodeInfo.get().getInternalNode();
            assignment.put(chosenNode, split);
            assignmentStats.addAssignedSplit(chosenNode, splitWeight);
        } else {
            splitWaitingForAnyNode = true;
        }
    }
    ListenableFuture<?> blocked = splitWaitingForAnyNode ? toWhenHasSplitQueueSpaceFuture(existingTasks, calculateLowWatermark(maxPendingSplitsWeightPerTask)) : immediateFuture(null);
    return new SplitPlacementResult(blocked, assignment.build());
}

Example 9 with SplitContext

use of com.facebook.presto.spi.SplitContext in project presto by prestodb.

the class DeltaPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, ConnectorTableLayoutHandle layout, List<ColumnHandle> columns, SplitContext splitContext) {
    DeltaSplit deltaSplit = (DeltaSplit) split;
    DeltaTableLayoutHandle deltaTableLayoutHandle = (DeltaTableLayoutHandle) layout;
    DeltaTableHandle deltaTableHandle = deltaTableLayoutHandle.getTable();
    HdfsContext hdfsContext = new HdfsContext(session, deltaSplit.getSchema(), deltaSplit.getTable(), deltaSplit.getFilePath(), false);
    Path filePath = new Path(deltaSplit.getFilePath());
    List<DeltaColumnHandle> deltaColumnHandles = columns.stream().map(DeltaColumnHandle.class::cast).collect(Collectors.toList());
    List<DeltaColumnHandle> regularColumnHandles = deltaColumnHandles.stream().filter(columnHandle -> columnHandle.getColumnType() != PARTITION).collect(Collectors.toList());
    ConnectorPageSource dataPageSource = createParquetPageSource(hdfsEnvironment, session.getUser(), hdfsEnvironment.getConfiguration(hdfsContext, filePath), filePath, deltaSplit.getStart(), deltaSplit.getLength(), deltaSplit.getFileSize(), regularColumnHandles, deltaTableHandle.toSchemaTableName(), getParquetMaxReadBlockSize(session), isParquetBatchReadsEnabled(session), isParquetBatchReaderVerificationEnabled(session), typeManager, deltaTableLayoutHandle.getPredicate(), fileFormatDataSourceStats, false);
    return new DeltaPageSource(deltaColumnHandles, convertPartitionValues(deltaColumnHandles, deltaSplit.getPartitionValues()), dataPageSource);
}

Example 10 with SplitContext

use of com.facebook.presto.spi.SplitContext in project presto by prestodb.

the class HivePageSourceProvider method shouldSkipPartition.

private static boolean shouldSkipPartition(TypeManager typeManager, HiveTableLayoutHandle hiveLayout, DateTimeZone hiveStorageTimeZone, HiveSplit hiveSplit, SplitContext splitContext) {
    List<HiveColumnHandle> partitionColumns = hiveLayout.getPartitionColumns();
    List<Type> partitionTypes = partitionColumns.stream().map(column -> typeManager.getType(column.getTypeSignature())).collect(toList());
    List<HivePartitionKey> partitionKeys = hiveSplit.getPartitionKeys();
    if (!splitContext.getDynamicFilterPredicate().isPresent() || hiveSplit.getPartitionKeys().isEmpty() || partitionColumns.isEmpty() || partitionColumns.size() != partitionKeys.size()) {
        return false;
    }
    TupleDomain<ColumnHandle> dynamicFilter = splitContext.getDynamicFilterPredicate().get();
    Map<ColumnHandle, Domain> domains = dynamicFilter.getDomains().get();
    for (int i = 0; i < partitionKeys.size(); i++) {
        Type type = partitionTypes.get(i);
        HivePartitionKey hivePartitionKey = partitionKeys.get(i);
        HiveColumnHandle hiveColumnHandle = partitionColumns.get(i);
        Domain allowedDomain = domains.get(hiveColumnHandle);
        NullableValue value = parsePartitionValue(hivePartitionKey, type, hiveStorageTimeZone);
        if (allowedDomain != null && !allowedDomain.includesNullableValue(value.getValue())) {
            return true;
        }
    }
    return false;
}