Examples with RemoteSourceNode com.facebook.presto.sql.planner.plan.RemoteSourceNode used on opensource projects

Example 11 with RemoteSourceNode

use of com.facebook.presto.sql.planner.plan.RemoteSourceNode in project presto by prestodb.

the class PrestoSparkRddFactory method createSparkRdd.

public <T extends PrestoSparkTaskOutput> JavaPairRDD<MutablePartitionId, T> createSparkRdd(JavaSparkContext sparkContext, Session session, PlanFragment fragment, Map<PlanFragmentId, JavaPairRDD<MutablePartitionId, PrestoSparkMutableRow>> rddInputs, Map<PlanFragmentId, Broadcast<?>> broadcastInputs, PrestoSparkTaskExecutorFactoryProvider executorFactoryProvider, CollectionAccumulator<SerializedTaskInfo> taskInfoCollector, CollectionAccumulator<PrestoSparkShuffleStats> shuffleStatsCollector, TableWriteInfo tableWriteInfo, Class<T> outputType) {
    checkArgument(!fragment.getStageExecutionDescriptor().isStageGroupedExecution(), "unexpected grouped execution fragment: %s", fragment.getId());
    PartitioningHandle partitioning = fragment.getPartitioning();
    if (partitioning.equals(SCALED_WRITER_DISTRIBUTION)) {
        throw new PrestoException(NOT_SUPPORTED, "Automatic writers scaling is not supported by Presto on Spark");
    }
    checkArgument(!partitioning.equals(COORDINATOR_DISTRIBUTION), "COORDINATOR_DISTRIBUTION fragment must be run on the driver");
    checkArgument(!partitioning.equals(FIXED_BROADCAST_DISTRIBUTION), "FIXED_BROADCAST_DISTRIBUTION can only be set as an output partitioning scheme, and not as a fragment distribution");
    checkArgument(!partitioning.equals(FIXED_PASSTHROUGH_DISTRIBUTION), "FIXED_PASSTHROUGH_DISTRIBUTION can only be set as local exchange partitioning");
    // TODO: ARBITRARY_DISTRIBUTION is something very weird.
    // TODO: It doesn't have partitioning function, and it is never set as a fragment partitioning.
    // TODO: We should consider removing ARBITRARY_DISTRIBUTION.
    checkArgument(!partitioning.equals(ARBITRARY_DISTRIBUTION), "ARBITRARY_DISTRIBUTION is not expected to be set as a fragment distribution");
    if (partitioning.equals(SINGLE_DISTRIBUTION) || partitioning.equals(FIXED_HASH_DISTRIBUTION) || partitioning.equals(FIXED_ARBITRARY_DISTRIBUTION) || partitioning.equals(SOURCE_DISTRIBUTION) || partitioning.getConnectorId().isPresent()) {
        for (RemoteSourceNode remoteSource : fragment.getRemoteSourceNodes()) {
            if (remoteSource.isEnsureSourceOrdering() || remoteSource.getOrderingScheme().isPresent()) {
                throw new PrestoException(NOT_SUPPORTED, format("Order sensitive exchange is not supported by Presto on Spark. fragmentId: %s, sourceFragmentIds: %s", fragment.getId(), remoteSource.getSourceFragmentIds()));
            }
        }
        return createRdd(sparkContext, session, fragment, executorFactoryProvider, taskInfoCollector, shuffleStatsCollector, tableWriteInfo, rddInputs, broadcastInputs, outputType);
    } else {
        throw new IllegalArgumentException(format("Unexpected fragment partitioning %s, fragmentId: %s", partitioning, fragment.getId()));
    }
}