Examples with TaskSource com.facebook.presto.execution.TaskSource used on opensource projects

Example 1 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class PrestoSparkModule method setup.

@Override
protected void setup(Binder binder) {
    // configs
    // TODO: decouple configuration properties that don't make sense on Spark
    configBinder(binder).bindConfig(NodeSchedulerConfig.class);
    configBinder(binder).bindConfig(SimpleTtlNodeSelectorConfig.class);
    configBinder(binder).bindConfig(QueryManagerConfig.class);
    configBinder(binder).bindConfigGlobalDefaults(QueryManagerConfig.class, PrestoSparkSettingsRequirements::setDefaults);
    configBinder(binder).bindConfig(FeaturesConfig.class);
    configBinder(binder).bindConfigGlobalDefaults(FeaturesConfig.class, PrestoSparkSettingsRequirements::setDefaults);
    configBinder(binder).bindConfig(MemoryManagerConfig.class);
    configBinder(binder).bindConfig(TaskManagerConfig.class);
    configBinder(binder).bindConfig(TransactionManagerConfig.class);
    configBinder(binder).bindConfig(NodeMemoryConfig.class);
    configBinder(binder).bindConfig(WarningCollectorConfig.class);
    configBinder(binder).bindConfig(NodeSpillConfig.class);
    configBinder(binder).bindConfig(CompilerConfig.class);
    configBinder(binder).bindConfig(SqlEnvironmentConfig.class);
    configBinder(binder).bindConfig(StaticFunctionNamespaceStoreConfig.class);
    configBinder(binder).bindConfig(PrestoSparkConfig.class);
    configBinder(binder).bindConfig(TracingConfig.class);
    // json codecs
    jsonCodecBinder(binder).bindJsonCodec(ViewDefinition.class);
    jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(PrestoSparkTaskDescriptor.class);
    jsonCodecBinder(binder).bindJsonCodec(PlanFragment.class);
    jsonCodecBinder(binder).bindJsonCodec(TaskSource.class);
    jsonCodecBinder(binder).bindJsonCodec(TableCommitContext.class);
    jsonCodecBinder(binder).bindJsonCodec(ExplainAnalyzeContext.class);
    jsonCodecBinder(binder).bindJsonCodec(ExecutionFailureInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(StageInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(OperatorStats.class);
    jsonCodecBinder(binder).bindJsonCodec(QueryInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(PrestoSparkQueryStatusInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(PrestoSparkQueryData.class);
    jsonCodecBinder(binder).bindListJsonCodec(TaskMemoryReservationSummary.class);
    // smile codecs
    smileCodecBinder(binder).bindSmileCodec(TaskSource.class);
    smileCodecBinder(binder).bindSmileCodec(TaskInfo.class);
    PrestoSparkConfig prestoSparkConfig = buildConfigObject(PrestoSparkConfig.class);
    if (prestoSparkConfig.isSmileSerializationEnabled()) {
        binder.bind(new TypeLiteral<Codec<TaskSource>>() {
        }).to(new TypeLiteral<SmileCodec<TaskSource>>() {
        }).in(Scopes.SINGLETON);
        binder.bind(new TypeLiteral<Codec<TaskInfo>>() {
        }).to(new TypeLiteral<SmileCodec<TaskInfo>>() {
        }).in(Scopes.SINGLETON);
    } else {
        binder.bind(new TypeLiteral<Codec<TaskSource>>() {
        }).to(new TypeLiteral<JsonCodec<TaskSource>>() {
        }).in(Scopes.SINGLETON);
        binder.bind(new TypeLiteral<Codec<TaskInfo>>() {
        }).to(new TypeLiteral<JsonCodec<TaskInfo>>() {
        }).in(Scopes.SINGLETON);
    }
    // index manager
    binder.bind(IndexManager.class).in(Scopes.SINGLETON);
    // handle resolver
    binder.install(new HandleJsonModule());
    // plugin manager
    configBinder(binder).bindConfig(PluginManagerConfig.class);
    binder.bind(PluginManager.class).in(Scopes.SINGLETON);
    // catalog manager
    binder.bind(StaticCatalogStore.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(StaticCatalogStoreConfig.class);
    // catalog
    binder.bind(ConnectorManager.class).in(Scopes.SINGLETON);
    binder.bind(CatalogManager.class).in(Scopes.SINGLETON);
    // property managers
    binder.bind(SessionPropertyManager.class).toProvider(PrestoSparkSessionPropertyManagerProvider.class).in(Scopes.SINGLETON);
    binder.bind(SystemSessionProperties.class).in(Scopes.SINGLETON);
    binder.bind(PrestoSparkSessionProperties.class).in(Scopes.SINGLETON);
    binder.bind(SessionPropertyDefaults.class).in(Scopes.SINGLETON);
    binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON);
    binder.bind(TablePropertyManager.class).in(Scopes.SINGLETON);
    binder.bind(ColumnPropertyManager.class).in(Scopes.SINGLETON);
    binder.bind(AnalyzePropertyManager.class).in(Scopes.SINGLETON);
    binder.bind(QuerySessionSupplier.class).in(Scopes.SINGLETON);
    // block encodings
    binder.bind(BlockEncodingManager.class).in(Scopes.SINGLETON);
    binder.bind(BlockEncodingSerde.class).to(BlockEncodingManager.class).in(Scopes.SINGLETON);
    newSetBinder(binder, BlockEncoding.class);
    jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class);
    jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class);
    // metadata
    binder.bind(FunctionAndTypeManager.class).in(Scopes.SINGLETON);
    binder.bind(MetadataManager.class).in(Scopes.SINGLETON);
    binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON);
    binder.bind(StaticFunctionNamespaceStore.class).in(Scopes.SINGLETON);
    // type
    newSetBinder(binder, Type.class);
    binder.bind(TypeManager.class).to(FunctionAndTypeManager.class).in(Scopes.SINGLETON);
    jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
    // PageSorter
    binder.bind(PageSorter.class).to(PagesIndexPageSorter.class).in(Scopes.SINGLETON);
    // PageIndexer
    binder.bind(PagesIndex.Factory.class).to(PagesIndex.DefaultFactory.class);
    binder.bind(PageIndexerFactory.class).to(GroupByHashPageIndexerFactory.class).in(Scopes.SINGLETON);
    // compilers
    binder.bind(JoinFilterFunctionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinFilterFunctionCompiler.class).withGeneratedName();
    binder.bind(JoinCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinCompiler.class).withGeneratedName();
    binder.bind(OrderingCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(OrderingCompiler.class).withGeneratedName();
    binder.bind(LookupJoinOperators.class).in(Scopes.SINGLETON);
    binder.bind(DomainTranslator.class).to(RowExpressionDomainTranslator.class).in(Scopes.SINGLETON);
    binder.bind(PredicateCompiler.class).to(RowExpressionPredicateCompiler.class).in(Scopes.SINGLETON);
    binder.bind(DeterminismEvaluator.class).to(RowExpressionDeterminismEvaluator.class).in(Scopes.SINGLETON);
    binder.bind(ExpressionCompiler.class).in(Scopes.SINGLETON);
    binder.bind(PageFunctionCompiler.class).in(Scopes.SINGLETON);
    // split manager
    binder.bind(SplitManager.class).in(Scopes.SINGLETON);
    // partitioning provider manager
    binder.bind(PartitioningProviderManager.class).in(Scopes.SINGLETON);
    // executors
    ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("presto-spark-executor-%s"));
    binder.bind(Executor.class).toInstance(executor);
    binder.bind(ExecutorService.class).toInstance(executor);
    binder.bind(ScheduledExecutorService.class).toInstance(newScheduledThreadPool(0, daemonThreadsNamed("presto-spark-scheduled-executor-%s")));
    // task executor
    binder.bind(EmbedVersion.class).in(Scopes.SINGLETON);
    binder.bind(MultilevelSplitQueue.class).in(Scopes.SINGLETON);
    binder.bind(TaskExecutor.class).in(Scopes.SINGLETON);
    // data stream provider
    binder.bind(PageSourceManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSourceProvider.class).to(PageSourceManager.class).in(Scopes.SINGLETON);
    // connector distributed metadata manager
    binder.bind(ConnectorMetadataUpdaterManager.class).in(Scopes.SINGLETON);
    // page sink provider
    binder.bind(PageSinkManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSinkProvider.class).to(PageSinkManager.class).in(Scopes.SINGLETON);
    // query explainer
    binder.bind(QueryExplainer.class).in(Scopes.SINGLETON);
    // parser
    binder.bind(PlanChecker.class).in(Scopes.SINGLETON);
    binder.bind(SqlParser.class).in(Scopes.SINGLETON);
    binder.bind(SqlParserOptions.class).toInstance(sqlParserOptions);
    // planner
    binder.bind(PlanFragmenter.class).in(Scopes.SINGLETON);
    binder.bind(PlanOptimizers.class).in(Scopes.SINGLETON);
    binder.bind(ConnectorPlanOptimizerManager.class).in(Scopes.SINGLETON);
    binder.bind(LocalExecutionPlanner.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(FileFragmentResultCacheConfig.class);
    binder.bind(FragmentCacheStats.class).in(Scopes.SINGLETON);
    binder.bind(IndexJoinLookupStats.class).in(Scopes.SINGLETON);
    binder.bind(QueryIdGenerator.class).in(Scopes.SINGLETON);
    binder.bind(QueryPreparer.class).in(Scopes.SINGLETON);
    jsonBinder(binder).addKeySerializerBinding(VariableReferenceExpression.class).to(VariableReferenceExpressionSerializer.class);
    jsonBinder(binder).addKeyDeserializerBinding(VariableReferenceExpression.class).to(VariableReferenceExpressionDeserializer.class);
    // statistics calculator / cost calculator
    binder.install(new StatsCalculatorModule());
    binder.bind(CostCalculator.class).to(CostCalculatorUsingExchanges.class).in(Scopes.SINGLETON);
    binder.bind(CostCalculator.class).annotatedWith(CostCalculator.EstimatedExchanges.class).to(CostCalculatorWithEstimatedExchanges.class).in(Scopes.SINGLETON);
    binder.bind(CostComparator.class).in(Scopes.SINGLETON);
    // JMX (Do not export to the real MXBean server, as the Presto context may be created multiple times per JVM)
    binder.bind(MBeanServer.class).toInstance(new TestingMBeanServer());
    binder.bind(MBeanExporter.class).in(Scopes.SINGLETON);
    // spill
    binder.bind(SpillerFactory.class).to(GenericSpillerFactory.class).in(Scopes.SINGLETON);
    binder.bind(SingleStreamSpillerFactory.class).to(TempStorageSingleStreamSpillerFactory.class).in(Scopes.SINGLETON);
    binder.bind(PartitioningSpillerFactory.class).to(GenericPartitioningSpillerFactory.class).in(Scopes.SINGLETON);
    binder.bind(SpillerStats.class).in(Scopes.SINGLETON);
    binder.bind(StandaloneSpillerFactory.class).to(TempStorageStandaloneSpillerFactory.class).in(Scopes.SINGLETON);
    // monitoring
    jsonCodecBinder(binder).bindJsonCodec(OperatorInfo.class);
    binder.bind(QueryMonitor.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(QueryMonitorConfig.class);
    binder.bind(SplitMonitor.class).in(Scopes.SINGLETON);
    // Determine the NodeVersion
    ServerConfig serverConfig = buildConfigObject(ServerConfig.class);
    NodeVersion nodeVersion = new NodeVersion(serverConfig.getPrestoVersion());
    binder.bind(NodeVersion.class).toInstance(nodeVersion);
    // Support DDL statements. Helper class binds DDL statements to DataDefinitionTasks
    PrestoDataDefBindingHelper.bindDDLDefinitionTasks(binder);
    // TODO: Decouple node specific system tables
    binder.bind(QueryManager.class).to(PrestoSparkQueryManager.class).in(Scopes.SINGLETON);
    binder.bind(TaskManager.class).to(PrestoSparkTaskManager.class).in(Scopes.SINGLETON);
    binder.install(new SystemConnectorModule());
    // TODO: support explain analyze for Spark
    binder.bind(new TypeLiteral<Optional<ExplainAnalyzeContext>>() {
    }).toInstance(Optional.of(new ExplainAnalyzeContext((queryId) -> {
        throw new UnsupportedOperationException("explain analyze is not supported");
    })));
    // TODO: support CBO, supply real nodes count
    binder.bind(TaskCountEstimator.class).toInstance(new TaskCountEstimator(() -> 1000));
    // TODO: Decouple and remove: required by ConnectorManager
    binder.bind(InternalNodeManager.class).toInstance(new PrestoSparkInternalNodeManager());
    // TODO: Decouple and remove: required by PluginManager
    binder.bind(InternalResourceGroupManager.class).in(Scopes.SINGLETON);
    binder.bind(ResourceGroupManager.class).to(InternalResourceGroupManager.class);
    binder.bind(new TypeLiteral<ResourceGroupManager<?>>() {
    }).to(new TypeLiteral<InternalResourceGroupManager<?>>() {
    });
    binder.bind(LegacyResourceGroupConfigurationManager.class).in(Scopes.SINGLETON);
    binder.bind(ClusterMemoryPoolManager.class).toInstance(((poolId, listener) -> {
    }));
    binder.bind(QueryPrerequisitesManager.class).in(Scopes.SINGLETON);
    binder.bind(ResourceGroupService.class).to(NoopResourceGroupService.class).in(Scopes.SINGLETON);
    binder.bind(NodeTtlFetcherManager.class).to(ThrowingNodeTtlFetcherManager.class).in(Scopes.SINGLETON);
    binder.bind(ClusterTtlProviderManager.class).to(ThrowingClusterTtlProviderManager.class).in(Scopes.SINGLETON);
    // TODO: Decouple and remove: required by SessionPropertyDefaults, PluginManager, InternalResourceGroupManager, ConnectorManager
    configBinder(binder).bindConfig(NodeConfig.class);
    binder.bind(NodeInfo.class).in(Scopes.SINGLETON);
    // TODO: Decouple and remove: required by LocalExecutionPlanner, PlanFragmenter
    binder.bind(NodePartitioningManager.class).to(PrestoSparkNodePartitioningManager.class).in(Scopes.SINGLETON);
    // TODO: Decouple and remove: required by PluginManager
    install(new ServerSecurityModule());
    // spark specific
    binder.bind(SparkProcessType.class).toInstance(sparkProcessType);
    binder.bind(PrestoSparkExecutionExceptionFactory.class).in(Scopes.SINGLETON);
    binder.bind(PrestoSparkSettingsRequirements.class).in(Scopes.SINGLETON);
    binder.bind(PrestoSparkQueryPlanner.class).in(Scopes.SINGLETON);
    binder.bind(PrestoSparkPlanFragmenter.class).in(Scopes.SINGLETON);
    binder.bind(PrestoSparkRddFactory.class).in(Scopes.SINGLETON);
    binder.bind(PrestoSparkTaskExecutorFactory.class).in(Scopes.SINGLETON);
    binder.bind(PrestoSparkQueryExecutionFactory.class).in(Scopes.SINGLETON);
    binder.bind(PrestoSparkService.class).in(Scopes.SINGLETON);
    binder.bind(PrestoSparkBroadcastTableCacheManager.class).in(Scopes.SINGLETON);
    newSetBinder(binder, PrestoSparkServiceWaitTimeMetrics.class);
    // extra credentials and authenticator for Presto-on-Spark
    newSetBinder(binder, PrestoSparkCredentialsProvider.class);
    newSetBinder(binder, PrestoSparkAuthenticatorProvider.class);
}

Example 2 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class PrestoSparkTaskExecutorFactory method computeAllSplitsSize.

private static OptionalLong computeAllSplitsSize(List<TaskSource> taskSources) {
    long sum = 0;
    for (TaskSource taskSource : taskSources) {
        for (ScheduledSplit scheduledSplit : taskSource.getSplits()) {
            ConnectorSplit connectorSplit = scheduledSplit.getSplit().getConnectorSplit();
            if (!connectorSplit.getSplitSizeInBytes().isPresent()) {
                return OptionalLong.empty();
            }
            sum += connectorSplit.getSplitSizeInBytes().getAsLong();
        }
    }
    return OptionalLong.of(sum);
}

Example 3 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class PrestoSparkTaskExecutorFactory method getTaskSources.

private List<TaskSource> getTaskSources(Iterator<SerializedPrestoSparkTaskSource> serializedTaskSources) {
    long totalSerializedSizeInBytes = 0;
    ImmutableList.Builder<TaskSource> result = ImmutableList.builder();
    while (serializedTaskSources.hasNext()) {
        SerializedPrestoSparkTaskSource serializedTaskSource = serializedTaskSources.next();
        totalSerializedSizeInBytes += serializedTaskSource.getBytes().length;
        result.add(deserializeZstdCompressed(taskSourceCodec, serializedTaskSource.getBytes()));
    }
    log.info("Total serialized size of all task sources: %s", succinctBytes(totalSerializedSizeInBytes));
    return result.build();
}

Example 4 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class PrestoSparkTaskExecutorFactory method doCreate.

public <T extends PrestoSparkTaskOutput> IPrestoSparkTaskExecutor<T> doCreate(int partitionId, int attemptNumber, SerializedPrestoSparkTaskDescriptor serializedTaskDescriptor, Iterator<SerializedPrestoSparkTaskSource> serializedTaskSources, PrestoSparkTaskInputs inputs, CollectionAccumulator<SerializedTaskInfo> taskInfoCollector, CollectionAccumulator<PrestoSparkShuffleStats> shuffleStatsCollector, Class<T> outputType) {
    PrestoSparkTaskDescriptor taskDescriptor = taskDescriptorJsonCodec.fromJson(serializedTaskDescriptor.getBytes());
    ImmutableMap.Builder<String, TokenAuthenticator> extraAuthenticators = ImmutableMap.builder();
    authenticatorProviders.forEach(provider -> extraAuthenticators.putAll(provider.getTokenAuthenticators()));
    Session session = taskDescriptor.getSession().toSession(sessionPropertyManager, taskDescriptor.getExtraCredentials(), extraAuthenticators.build());
    PlanFragment fragment = taskDescriptor.getFragment();
    StageId stageId = new StageId(session.getQueryId(), fragment.getId().getId());
    // Clear the cache if the cache does not have broadcast table for current stageId.
    // We will only cache 1 HT at any time. If the stageId changes, we will drop the old cached HT
    prestoSparkBroadcastTableCacheManager.removeCachedTablesForStagesOtherThan(stageId);
    // TODO: include attemptId in taskId
    TaskId taskId = new TaskId(new StageExecutionId(stageId, 0), partitionId);
    List<TaskSource> taskSources = getTaskSources(serializedTaskSources);
    log.info("Task [%s] received %d splits.", taskId, taskSources.stream().mapToInt(taskSource -> taskSource.getSplits().size()).sum());
    OptionalLong totalSplitSize = computeAllSplitsSize(taskSources);
    if (totalSplitSize.isPresent()) {
        log.info("Total split size: %s bytes.", totalSplitSize.getAsLong());
    }
    // TODO: Remove this once we can display the plan on Spark UI.
    log.info(PlanPrinter.textPlanFragment(fragment, functionAndTypeManager, session, true));
    DataSize maxUserMemory = new DataSize(min(nodeMemoryConfig.getMaxQueryMemoryPerNode().toBytes(), getQueryMaxMemoryPerNode(session).toBytes()), BYTE);
    DataSize maxTotalMemory = new DataSize(min(nodeMemoryConfig.getMaxQueryTotalMemoryPerNode().toBytes(), getQueryMaxTotalMemoryPerNode(session).toBytes()), BYTE);
    DataSize maxBroadcastMemory = getSparkBroadcastJoinMaxMemoryOverride(session);
    if (maxBroadcastMemory == null) {
        maxBroadcastMemory = new DataSize(min(nodeMemoryConfig.getMaxQueryBroadcastMemory().toBytes(), getQueryMaxBroadcastMemory(session).toBytes()), BYTE);
    }
    MemoryPool memoryPool = new MemoryPool(new MemoryPoolId("spark-executor-memory-pool"), maxTotalMemory);
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(maxQuerySpillPerNode);
    QueryContext queryContext = new QueryContext(session.getQueryId(), maxUserMemory, maxTotalMemory, maxBroadcastMemory, maxRevocableMemory, memoryPool, new TestingGcMonitor(), notificationExecutor, yieldExecutor, maxQuerySpillPerNode, spillSpaceTracker, memoryReservationSummaryJsonCodec);
    queryContext.setVerboseExceededMemoryLimitErrorsEnabled(isVerboseExceededMemoryLimitErrorsEnabled(session));
    queryContext.setHeapDumpOnExceededMemoryLimitEnabled(isHeapDumpOnExceededMemoryLimitEnabled(session));
    String heapDumpFilePath = Paths.get(getHeapDumpFileDirectory(session), format("%s_%s.hprof", session.getQueryId().getId(), stageId.getId())).toString();
    queryContext.setHeapDumpFilePath(heapDumpFilePath);
    TaskStateMachine taskStateMachine = new TaskStateMachine(taskId, notificationExecutor);
    TaskContext taskContext = queryContext.addTaskContext(taskStateMachine, session, // Plan has to be retained only if verbose memory exceeded errors are requested
    isVerboseExceededMemoryLimitErrorsEnabled(session) ? Optional.of(fragment.getRoot()) : Optional.empty(), perOperatorCpuTimerEnabled, cpuTimerEnabled, perOperatorAllocationTrackingEnabled, allocationTrackingEnabled, false);
    final double memoryRevokingThreshold = getMemoryRevokingThreshold(session);
    final double memoryRevokingTarget = getMemoryRevokingTarget(session);
    checkArgument(memoryRevokingTarget <= memoryRevokingThreshold, "memoryRevokingTarget should be less than or equal memoryRevokingThreshold, but got %s and %s respectively", memoryRevokingTarget, memoryRevokingThreshold);
    if (isSpillEnabled(session)) {
        memoryPool.addListener((pool, queryId, totalMemoryReservationBytes) -> {
            if (totalMemoryReservationBytes > queryContext.getPeakNodeTotalMemory()) {
                queryContext.setPeakNodeTotalMemory(totalMemoryReservationBytes);
            }
            if (totalMemoryReservationBytes > pool.getMaxBytes() * memoryRevokingThreshold && memoryRevokeRequestInProgress.compareAndSet(false, true)) {
                memoryRevocationExecutor.execute(() -> {
                    try {
                        AtomicLong remainingBytesToRevoke = new AtomicLong(totalMemoryReservationBytes - (long) (memoryRevokingTarget * pool.getMaxBytes()));
                        remainingBytesToRevoke.addAndGet(-MemoryRevokingSchedulerUtils.getMemoryAlreadyBeingRevoked(ImmutableList.of(taskContext), remainingBytesToRevoke.get()));
                        taskContext.accept(new VoidTraversingQueryContextVisitor<AtomicLong>() {

                            @Override
                            public Void visitOperatorContext(OperatorContext operatorContext, AtomicLong remainingBytesToRevoke) {
                                if (remainingBytesToRevoke.get() > 0) {
                                    long revokedBytes = operatorContext.requestMemoryRevoking();
                                    if (revokedBytes > 0) {
                                        memoryRevokePending.set(true);
                                        remainingBytesToRevoke.addAndGet(-revokedBytes);
                                    }
                                }
                                return null;
                            }
                        }, remainingBytesToRevoke);
                        memoryRevokeRequestInProgress.set(false);
                    } catch (Exception e) {
                        log.error(e, "Error requesting memory revoking");
                    }
                });
            }
            // Get the latest memory reservation info since it might have changed due to revoke
            long totalReservedMemory = pool.getQueryMemoryReservation(queryId) + pool.getQueryRevocableMemoryReservation(queryId);
            // If total memory usage is over maxTotalMemory and memory revoke request is not pending, fail the query with EXCEEDED_MEMORY_LIMIT error
            if (totalReservedMemory > maxTotalMemory.toBytes() && !memoryRevokeRequestInProgress.get() && !isMemoryRevokePending(taskContext)) {
                throw exceededLocalTotalMemoryLimit(maxTotalMemory, queryContext.getAdditionalFailureInfo(totalReservedMemory, 0) + format("Total reserved memory: %s, Total revocable memory: %s", succinctBytes(pool.getQueryMemoryReservation(queryId)), succinctBytes(pool.getQueryRevocableMemoryReservation(queryId))), isHeapDumpOnExceededMemoryLimitEnabled(session), Optional.ofNullable(heapDumpFilePath));
            }
        });
    }
    ImmutableMap.Builder<PlanNodeId, List<PrestoSparkShuffleInput>> shuffleInputs = ImmutableMap.builder();
    ImmutableMap.Builder<PlanNodeId, List<java.util.Iterator<PrestoSparkSerializedPage>>> pageInputs = ImmutableMap.builder();
    ImmutableMap.Builder<PlanNodeId, List<?>> broadcastInputs = ImmutableMap.builder();
    for (RemoteSourceNode remoteSource : fragment.getRemoteSourceNodes()) {
        List<PrestoSparkShuffleInput> remoteSourceRowInputs = new ArrayList<>();
        List<java.util.Iterator<PrestoSparkSerializedPage>> remoteSourcePageInputs = new ArrayList<>();
        List<List<?>> broadcastInputsList = new ArrayList<>();
        for (PlanFragmentId sourceFragmentId : remoteSource.getSourceFragmentIds()) {
            Iterator<Tuple2<MutablePartitionId, PrestoSparkMutableRow>> shuffleInput = inputs.getShuffleInputs().get(sourceFragmentId.toString());
            Broadcast<?> broadcastInput = inputs.getBroadcastInputs().get(sourceFragmentId.toString());
            List<PrestoSparkSerializedPage> inMemoryInput = inputs.getInMemoryInputs().get(sourceFragmentId.toString());
            if (shuffleInput != null) {
                checkArgument(broadcastInput == null, "single remote source is not expected to accept different kind of inputs");
                checkArgument(inMemoryInput == null, "single remote source is not expected to accept different kind of inputs");
                remoteSourceRowInputs.add(new PrestoSparkShuffleInput(sourceFragmentId.getId(), shuffleInput));
                continue;
            }
            if (broadcastInput != null) {
                checkArgument(inMemoryInput == null, "single remote source is not expected to accept different kind of inputs");
                // TODO: Enable NullifyingIterator once migrated to one task per JVM model
                // NullifyingIterator removes element from the list upon return
                // This allows GC to gradually reclaim memory
                // remoteSourcePageInputs.add(getNullifyingIterator(broadcastInput.value()));
                broadcastInputsList.add((List<?>) broadcastInput.value());
                continue;
            }
            if (inMemoryInput != null) {
                // for inmemory inputs pages can be released incrementally to save memory
                remoteSourcePageInputs.add(getNullifyingIterator(inMemoryInput));
                continue;
            }
            throw new IllegalArgumentException("Input not found for sourceFragmentId: " + sourceFragmentId);
        }
        if (!remoteSourceRowInputs.isEmpty()) {
            shuffleInputs.put(remoteSource.getId(), remoteSourceRowInputs);
        }
        if (!remoteSourcePageInputs.isEmpty()) {
            pageInputs.put(remoteSource.getId(), remoteSourcePageInputs);
        }
        if (!broadcastInputsList.isEmpty()) {
            broadcastInputs.put(remoteSource.getId(), broadcastInputsList);
        }
    }
    OutputBufferMemoryManager memoryManager = new OutputBufferMemoryManager(sinkMaxBufferSize.toBytes(), () -> queryContext.getTaskContextByTaskId(taskId).localSystemMemoryContext(), notificationExecutor);
    Optional<OutputPartitioning> preDeterminedPartition = Optional.empty();
    if (fragment.getPartitioningScheme().getPartitioning().getHandle().equals(FIXED_ARBITRARY_DISTRIBUTION)) {
        int partitionCount = getHashPartitionCount(session);
        preDeterminedPartition = Optional.of(new OutputPartitioning(new PreDeterminedPartitionFunction(partitionId % partitionCount, partitionCount), ImmutableList.of(), ImmutableList.of(), false, OptionalInt.empty()));
    }
    TempDataOperationContext tempDataOperationContext = new TempDataOperationContext(session.getSource(), session.getQueryId().getId(), session.getClientInfo(), Optional.of(session.getClientTags()), session.getIdentity());
    TempStorage tempStorage = tempStorageManager.getTempStorage(storageBasedBroadcastJoinStorage);
    Output<T> output = configureOutput(outputType, blockEncodingManager, memoryManager, getShuffleOutputTargetAverageRowSize(session), preDeterminedPartition, tempStorage, tempDataOperationContext, getStorageBasedBroadcastJoinWriteBufferSize(session));
    PrestoSparkOutputBuffer<?> outputBuffer = output.getOutputBuffer();
    LocalExecutionPlan localExecutionPlan = localExecutionPlanner.plan(taskContext, fragment.getRoot(), fragment.getPartitioningScheme(), fragment.getStageExecutionDescriptor(), fragment.getTableScanSchedulingOrder(), output.getOutputFactory(), new PrestoSparkRemoteSourceFactory(blockEncodingManager, shuffleInputs.build(), pageInputs.build(), broadcastInputs.build(), partitionId, shuffleStatsCollector, tempStorage, tempDataOperationContext, prestoSparkBroadcastTableCacheManager, stageId), taskDescriptor.getTableWriteInfo(), true);
    taskStateMachine.addStateChangeListener(state -> {
        if (state.isDone()) {
            outputBuffer.setNoMoreRows();
        }
    });
    PrestoSparkTaskExecution taskExecution = new PrestoSparkTaskExecution(taskStateMachine, taskContext, localExecutionPlan, taskExecutor, splitMonitor, notificationExecutor, memoryUpdateExecutor);
    taskExecution.start(taskSources);
    return new PrestoSparkTaskExecutor<>(taskContext, taskStateMachine, output.getOutputSupplier(), taskInfoCodec, taskInfoCollector, shuffleStatsCollector, executionExceptionFactory, output.getOutputBufferType(), outputBuffer, tempStorage, tempDataOperationContext);
}

Example 5 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class HttpRemoteTask method sendUpdate.

private synchronized void sendUpdate() {
    TaskStatus taskStatus = getTaskStatus();
    // don't update if the task hasn't been started yet or if it is already finished
    if (!started.get() || !needsUpdate.get() || taskStatus.getState().isDone()) {
        return;
    }
    // if there is a request already running, wait for it to complete
    if (this.currentRequest != null && !this.currentRequest.isDone()) {
        return;
    }
    // if throttled due to error, asynchronously wait for timeout and try again
    ListenableFuture<?> errorRateLimit = updateErrorTracker.acquireRequestPermit();
    if (!errorRateLimit.isDone()) {
        errorRateLimit.addListener(this::sendUpdate, executor);
        return;
    }
    List<TaskSource> sources = getSources();
    Optional<byte[]> fragment = sendPlan.get() ? Optional.of(planFragment.toBytes(planFragmentCodec)) : Optional.empty();
    Optional<TableWriteInfo> writeInfo = sendPlan.get() ? Optional.of(tableWriteInfo) : Optional.empty();
    TaskUpdateRequest updateRequest = new TaskUpdateRequest(session.toSessionRepresentation(), session.getIdentity().getExtraCredentials(), fragment, sources, outputBuffers.get(), writeInfo);
    byte[] taskUpdateRequestJson = taskUpdateRequestCodec.toBytes(updateRequest);
    taskUpdateRequestSize.add(taskUpdateRequestJson.length);
    if (taskUpdateRequestJson.length > maxTaskUpdateSizeInBytes) {
        failTask(new PrestoException(EXCEEDED_TASK_UPDATE_SIZE_LIMIT, format("TaskUpdate size of %d Bytes has exceeded the limit of %d Bytes", taskUpdateRequestJson.length, maxTaskUpdateSizeInBytes)));
    }
    if (fragment.isPresent()) {
        stats.updateWithPlanSize(taskUpdateRequestJson.length);
    } else {
        if (ThreadLocalRandom.current().nextDouble() < UPDATE_WITHOUT_PLAN_STATS_SAMPLE_RATE) {
            // This is to keep track of the task update size even when the plan fragment is NOT present
            stats.updateWithoutPlanSize(taskUpdateRequestJson.length);
        }
    }
    HttpUriBuilder uriBuilder = getHttpUriBuilder(taskStatus);
    Request request = setContentTypeHeaders(binaryTransportEnabled, preparePost()).setUri(uriBuilder.build()).setBodyGenerator(createStaticBodyGenerator(taskUpdateRequestJson)).build();
    ResponseHandler responseHandler;
    if (binaryTransportEnabled) {
        responseHandler = createFullSmileResponseHandler((SmileCodec<TaskInfo>) taskInfoCodec);
    } else {
        responseHandler = createAdaptingJsonResponseHandler((JsonCodec<TaskInfo>) taskInfoCodec);
    }
    updateErrorTracker.startRequest();
    ListenableFuture<BaseResponse<TaskInfo>> future = httpClient.executeAsync(request, responseHandler);
    currentRequest = future;
    currentRequestStartNanos = System.nanoTime();
    // The needsUpdate flag needs to be set to false BEFORE adding the Future callback since callback might change the flag value
    // and does so without grabbing the instance lock.
    needsUpdate.set(false);
    Futures.addCallback(future, new SimpleHttpResponseHandler<>(new UpdateResponseHandler(sources), request.getUri(), stats.getHttpResponseStats(), REMOTE_TASK_ERROR), executor);
}