Search in sources :

Example 1 with NodePartitioningManager

use of io.prestosql.sql.planner.NodePartitioningManager in project hetu-core by openlookeng.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    Metadata metadata = createTestMetadataManager();
    PageSourceManager pageSourceManager = new PageSourceManager();
    HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
    NodeInfo nodeInfo = new NodeInfo("test");
    FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
    SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
    StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
    HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
    return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}
Also used : NodeInfo(io.airlift.node.NodeInfo) SqlParser(io.prestosql.sql.parser.SqlParser) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) TEST_TABLE_HANDLE(io.prestosql.testing.TestingHandles.TEST_TABLE_HANDLE) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) ObjectMapperProvider(io.airlift.json.ObjectMapperProvider) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Partitioning(io.prestosql.sql.planner.Partitioning) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) MockExchangeClientSupplier(io.prestosql.execution.TestSqlTaskManager.MockExchangeClientSupplier) EventListenerManager(io.prestosql.eventlistener.EventListenerManager) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) CatalogName(io.prestosql.spi.connector.CatalogName) TableScanNode(io.prestosql.spi.plan.TableScanNode) UUID(java.util.UUID) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) Metadata(io.prestosql.metadata.Metadata) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) List(java.util.List) SOURCE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) PageSinkManager(io.prestosql.split.PageSinkManager) Optional(java.util.Optional) TEST_SESSION(io.prestosql.SessionTestUtils.TEST_SESSION) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) SINGLE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) StatsAndCosts(io.prestosql.cost.StatsAndCosts) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) Split(io.prestosql.metadata.Split) OutputBuffers(io.prestosql.execution.buffer.OutputBuffers) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) OptionalInt(java.util.OptionalInt) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) TestingSplit(io.prestosql.testing.TestingSplit) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) PageSourceManager(io.prestosql.split.PageSourceManager) TestingColumnHandle(io.prestosql.testing.TestingMetadata.TestingColumnHandle) PagesIndex(io.prestosql.operator.PagesIndex) Symbol(io.prestosql.spi.plan.Symbol) PlanFragment(io.prestosql.sql.planner.PlanFragment) StageExecutionDescriptor.ungroupedExecution(io.prestosql.operator.StageExecutionDescriptor.ungroupedExecution) IndexManager(io.prestosql.index.IndexManager) SplitMonitor(io.prestosql.event.SplitMonitor) FinalizerService(io.prestosql.util.FinalizerService) CubeManager(io.prestosql.cube.CubeManager) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) Metadata(io.prestosql.metadata.Metadata) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) PagesIndex(io.prestosql.operator.PagesIndex) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) PageSourceManager(io.prestosql.split.PageSourceManager) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) PageSinkManager(io.prestosql.split.PageSinkManager) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) MockExchangeClientSupplier(io.prestosql.execution.TestSqlTaskManager.MockExchangeClientSupplier) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) SqlParser(io.prestosql.sql.parser.SqlParser) CubeManager(io.prestosql.cube.CubeManager) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) IndexManager(io.prestosql.index.IndexManager) FinalizerService(io.prestosql.util.FinalizerService) NodeInfo(io.airlift.node.NodeInfo) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler)

Example 2 with NodePartitioningManager

use of io.prestosql.sql.planner.NodePartitioningManager in project hetu-core by openlookeng.

the class SqlQueryScheduler method createStages.

private List<SqlStageExecution> createStages(ExchangeLocationsConsumer parent, AtomicInteger nextStageId, LocationFactory locationFactory, StageExecutionPlan plan, NodeScheduler nodeScheduler, RemoteTaskFactory remoteTaskFactory, Session session, int splitBatchSize, BiFunction<PartitioningHandle, Integer, NodePartitionMap> partitioningCache, NodePartitioningManager nodePartitioningManager, ExecutorService queryExecutor, ScheduledExecutorService schedulerExecutor, FailureDetector failureDetector, NodeTaskMap nodeTaskMap, ImmutableMap.Builder<StageId, StageScheduler> stageSchedulers, ImmutableMap.Builder<StageId, StageLinkage> stageLinkages, boolean isSnapshotEnabled, QuerySnapshotManager snapshotManager, Map<StageId, Integer> stageTaskCounts) {
    ImmutableList.Builder<SqlStageExecution> localStages = ImmutableList.builder();
    StageId stageId = new StageId(queryStateMachine.getQueryId(), nextStageId.getAndIncrement());
    SqlStageExecution stageExecution = createSqlStageExecution(stageId, locationFactory.createStageLocation(stageId), plan.getFragment(), plan.getTables(), remoteTaskFactory, session, summarizeTaskInfo, nodeTaskMap, queryExecutor, failureDetector, schedulerStats, dynamicFilterService, snapshotManager);
    localStages.add(stageExecution);
    Optional<int[]> bucketToPartition;
    PartitioningHandle partitioningHandle = plan.getFragment().getPartitioning();
    boolean keepConsumerOnFeederNodes = !plan.getFragment().getFeederCTEId().isPresent() && plan.getFragment().getFeederCTEParentId().isPresent();
    if (partitioningHandle.equals(SOURCE_DISTRIBUTION)) {
        // nodes are selected dynamically based on the constraints of the splits and the system load
        Entry<PlanNodeId, SplitSource> entry = Iterables.getOnlyElement(plan.getSplitSources().entrySet());
        PlanNodeId planNodeId = entry.getKey();
        SplitSource splitSource = entry.getValue();
        CatalogName catalogName = splitSource.getCatalogName();
        if (isInternalSystemConnector(catalogName)) {
            catalogName = null;
        }
        NodeSelector nodeSelector = nodeScheduler.createNodeSelector(catalogName, keepConsumerOnFeederNodes, feederScheduledNodes);
        if (isSnapshotEnabled) {
            // When snapshot is enabled, then no task can be added after the query started running,
            // otherwise assumptions about how many "input channels" may be broken.
            nodeSelector.lockDownNodes();
        }
        SplitPlacementPolicy placementPolicy = new DynamicSplitPlacementPolicy(nodeSelector, stageExecution::getAllTasks);
        checkArgument(!plan.getFragment().getStageExecutionDescriptor().isStageGroupedExecution());
        stageSchedulers.put(stageId, newSourcePartitionedSchedulerAsStageScheduler(stageExecution, planNodeId, splitSource, placementPolicy, splitBatchSize, session, heuristicIndexerManager));
        bucketToPartition = Optional.of(new int[1]);
    } else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) {
        bucketToPartition = Optional.of(new int[1]);
    } else {
        Map<PlanNodeId, SplitSource> splitSources = plan.getSplitSources();
        if (!splitSources.isEmpty()) {
            // contains local source
            List<PlanNodeId> schedulingOrder = plan.getFragment().getPartitionedSources();
            CatalogName catalogName = partitioningHandle.getConnectorId().orElseThrow(IllegalStateException::new);
            List<ConnectorPartitionHandle> connectorPartitionHandles;
            boolean groupedExecutionForStage = plan.getFragment().getStageExecutionDescriptor().isStageGroupedExecution();
            if (groupedExecutionForStage) {
                connectorPartitionHandles = nodePartitioningManager.listPartitionHandles(session, partitioningHandle);
                checkState(!ImmutableList.of(NOT_PARTITIONED).equals(connectorPartitionHandles));
            } else {
                connectorPartitionHandles = ImmutableList.of(NOT_PARTITIONED);
            }
            BucketNodeMap bucketNodeMap;
            List<InternalNode> stageNodeList;
            if (plan.getFragment().getRemoteSourceNodes().stream().allMatch(node -> node.getExchangeType() == REPLICATE)) {
                // no remote source
                boolean dynamicLifespanSchedule = plan.getFragment().getStageExecutionDescriptor().isDynamicLifespanSchedule();
                if (isSnapshotEnabled) {
                    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(catalogName, keepConsumerOnFeederNodes, feederScheduledNodes);
                    int nodeCount;
                    if (stageTaskCounts != null) {
                        // Resuming: need to create same number of tasks as old stage.
                        nodeCount = stageTaskCounts.get(stageId);
                    } else {
                        // Scheduling: reserve some nodes for resuming
                        nodeCount = calculateTaskCount(nodeSelector.selectableNodeCount());
                    }
                    stageNodeList = new ArrayList<>(nodeSelector.selectRandomNodes(nodeCount));
                    checkCondition(stageNodeList.size() == nodeCount, NO_NODES_AVAILABLE, "Snapshot: not enough worker nodes to resume expected number of tasks: " + nodeCount);
                    // Make sure bucketNodeMap uses the same node list
                    bucketNodeMap = nodePartitioningManager.getBucketNodeMap(session, partitioningHandle, dynamicLifespanSchedule, stageNodeList);
                } else {
                    bucketNodeMap = nodePartitioningManager.getBucketNodeMap(session, partitioningHandle, dynamicLifespanSchedule);
                    stageNodeList = new ArrayList<>(nodeScheduler.createNodeSelector(catalogName, keepConsumerOnFeederNodes, feederScheduledNodes).allNodes());
                }
                // verify execution is consistent with planner's decision on dynamic lifespan schedule
                verify(bucketNodeMap.isDynamic() == dynamicLifespanSchedule);
                Collections.shuffle(stageNodeList);
                bucketToPartition = Optional.empty();
            } else {
                // cannot use dynamic lifespan schedule
                verify(!plan.getFragment().getStageExecutionDescriptor().isDynamicLifespanSchedule());
                // remote source requires nodePartitionMap
                NodePartitionMap nodePartitionMap = partitioningCache.apply(plan.getFragment().getPartitioning(), stageTaskCounts == null ? null : stageTaskCounts.get(stageId));
                if (groupedExecutionForStage) {
                    checkState(connectorPartitionHandles.size() == nodePartitionMap.getBucketToPartition().length);
                }
                stageNodeList = nodePartitionMap.getPartitionToNode();
                bucketNodeMap = nodePartitionMap.asBucketNodeMap();
                bucketToPartition = Optional.of(nodePartitionMap.getBucketToPartition());
            }
            stageSchedulers.put(stageId, new FixedSourcePartitionedScheduler(stageExecution, splitSources, plan.getFragment().getStageExecutionDescriptor(), schedulingOrder, stageNodeList, bucketNodeMap, splitBatchSize, getConcurrentLifespansPerNode(session), nodeScheduler.createNodeSelector(catalogName, keepConsumerOnFeederNodes, feederScheduledNodes), connectorPartitionHandles, session, heuristicIndexerManager));
        } else {
            // all sources are remote
            NodePartitionMap nodePartitionMap = partitioningCache.apply(plan.getFragment().getPartitioning(), stageTaskCounts == null ? null : stageTaskCounts.get(stageId));
            List<InternalNode> partitionToNode = nodePartitionMap.getPartitionToNode();
            // todo this should asynchronously wait a standard timeout period before failing
            checkCondition(!partitionToNode.isEmpty(), NO_NODES_AVAILABLE, "No worker nodes available");
            stageSchedulers.put(stageId, new FixedCountScheduler(stageExecution, partitionToNode));
            bucketToPartition = Optional.of(nodePartitionMap.getBucketToPartition());
        }
    }
    ImmutableSet.Builder<SqlStageExecution> childStagesBuilder = ImmutableSet.builder();
    for (StageExecutionPlan subStagePlan : plan.getSubStages()) {
        if (visitedPlanFrags.contains(subStagePlan.getFragment().getId())) {
            continue;
        }
        visitedPlanFrags.add(subStagePlan.getFragment().getId());
        List<SqlStageExecution> subTree = createStages(stageExecution::addExchangeLocations, nextStageId, locationFactory, subStagePlan.withBucketToPartition(bucketToPartition), nodeScheduler, remoteTaskFactory, session, splitBatchSize, partitioningCache, nodePartitioningManager, queryExecutor, schedulerExecutor, failureDetector, nodeTaskMap, stageSchedulers, stageLinkages, isSnapshotEnabled, snapshotManager, stageTaskCounts);
        localStages.addAll(subTree);
        SqlStageExecution childStage = subTree.get(0);
        childStagesBuilder.add(childStage);
        Optional<RemoteSourceNode> parentNode = plan.getFragment().getRemoteSourceNodes().stream().filter(x -> x.getSourceFragmentIds().contains(childStage.getFragment().getId())).findAny();
        checkArgument(parentNode.isPresent(), "Couldn't find parent of a CTE node");
        childStage.setParentId(parentNode.get().getId());
    }
    Set<SqlStageExecution> childStages = childStagesBuilder.build();
    stageExecution.addStateChangeListener(newState -> {
        if (newState.isDone() && newState != StageState.RESCHEDULING) {
            // Snapshot: For "rescheduling", tasks are already cancelled (for resume)
            childStages.forEach(SqlStageExecution::cancel);
        }
    });
    stageLinkages.put(stageId, new StageLinkage(plan.getFragment().getId(), parent, childStages));
    if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) {
        Supplier<Collection<TaskStatus>> sourceTasksProvider = () -> childStages.stream().map(SqlStageExecution::getAllTasks).flatMap(Collection::stream).map(RemoteTask::getTaskStatus).collect(toList());
        Supplier<Collection<TaskStatus>> writerTasksProvider = () -> stageExecution.getAllTasks().stream().map(RemoteTask::getTaskStatus).collect(toList());
        ScaledWriterScheduler scheduler = new ScaledWriterScheduler(stageExecution, sourceTasksProvider, writerTasksProvider, nodeScheduler.createNodeSelector(null, keepConsumerOnFeederNodes, feederScheduledNodes), schedulerExecutor, getWriterMinSize(session), isSnapshotEnabled, stageTaskCounts != null ? stageTaskCounts.get(stageId) : null);
        whenAllStages(childStages, StageState::isDone).addListener(scheduler::finish, directExecutor());
        stageSchedulers.put(stageId, scheduler);
    }
    return localStages.build();
}
Also used : CANCELED(io.prestosql.execution.StageState.CANCELED) SCHEDULED(io.prestosql.execution.StageState.SCHEDULED) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) NO_NODES_AVAILABLE(io.prestosql.spi.StandardErrorCode.NO_NODES_AVAILABLE) FIXED_BROADCAST_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.FIXED_BROADCAST_DISTRIBUTION) StageExecutionPlan(io.prestosql.sql.planner.StageExecutionPlan) Map(java.util.Map) SystemSessionProperties.getWriterMinSize(io.prestosql.SystemSessionProperties.getWriterMinSize) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) TaskStatus(io.prestosql.execution.TaskStatus) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) LocationFactory(io.prestosql.execution.LocationFactory) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) StageState(io.prestosql.execution.StageState) ConnectorPartitionHandle(io.prestosql.spi.connector.ConnectorPartitionHandle) GENERIC_INTERNAL_ERROR(io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) SetThreadName(io.airlift.concurrent.SetThreadName) Iterables(com.google.common.collect.Iterables) RESUMABLE_FAILURE(io.prestosql.execution.StageState.RESUMABLE_FAILURE) Supplier(java.util.function.Supplier) SCALED_WRITER_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SCALED_WRITER_DISTRIBUTION) QueryStateMachine(io.prestosql.execution.QueryStateMachine) ArrayList(java.util.ArrayList) CatalogName.isInternalSystemConnector(io.prestosql.spi.connector.CatalogName.isInternalSystemConnector) Session(io.prestosql.Session) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) NodeTaskMap(io.prestosql.execution.NodeTaskMap) SplitSource(io.prestosql.split.SplitSource) FINISHED(io.prestosql.execution.StageState.FINISHED) StageId(io.prestosql.execution.StageId) InternalNode(io.prestosql.metadata.InternalNode) Sets.newConcurrentHashSet(com.google.common.collect.Sets.newConcurrentHashSet) QuerySnapshotManager(io.prestosql.snapshot.QuerySnapshotManager) ResourceGroupInfo(io.prestosql.server.ResourceGroupInfo) PartitioningHandle(io.prestosql.sql.planner.PartitioningHandle) QueryState(io.prestosql.execution.QueryState) NodePartitionMap(io.prestosql.sql.planner.NodePartitionMap) SqlStageExecution.createSqlStageExecution(io.prestosql.execution.SqlStageExecution.createSqlStageExecution) SqlStageExecution(io.prestosql.execution.SqlStageExecution) ABORTED(io.prestosql.execution.StageState.ABORTED) FailureDetector(io.prestosql.failuredetector.FailureDetector) RemoteTask(io.prestosql.execution.RemoteTask) FAILED(io.prestosql.execution.StageState.FAILED) SystemSessionProperties(io.prestosql.SystemSessionProperties) BiFunction(java.util.function.BiFunction) SettableFuture(com.google.common.util.concurrent.SettableFuture) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(io.prestosql.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) Duration(io.airlift.units.Duration) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) URI(java.net.URI) Collectors.toSet(java.util.stream.Collectors.toSet) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collection(java.util.Collection) CatalogName(io.prestosql.spi.connector.CatalogName) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) RemoteTaskFactory(io.prestosql.execution.RemoteTaskFactory) UUID(java.util.UUID) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) SOURCE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) StageInfo(io.prestosql.execution.StageInfo) Entry(java.util.Map.Entry) HttpUriBuilder.uriBuilderFrom(io.airlift.http.client.HttpUriBuilder.uriBuilderFrom) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) MoreFutures.whenAnyComplete(io.airlift.concurrent.MoreFutures.whenAnyComplete) BasicStageStats(io.prestosql.execution.BasicStageStats) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) NOT_PARTITIONED(io.prestosql.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) RUNNING(io.prestosql.execution.StageState.RUNNING) TaskId(io.prestosql.execution.TaskId) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) Logger(io.airlift.log.Logger) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) HashMap(java.util.HashMap) OutputBuffers(io.prestosql.execution.buffer.OutputBuffers) TaskLocation(io.prestosql.operator.TaskLocation) HashSet(java.util.HashSet) SnapshotConfig.calculateTaskCount(io.prestosql.snapshot.SnapshotConfig.calculateTaskCount) ImmutableList(com.google.common.collect.ImmutableList) SystemSessionProperties.isReuseTableScanEnabled(io.prestosql.SystemSessionProperties.isReuseTableScanEnabled) OutputBufferId(io.prestosql.execution.buffer.OutputBuffers.OutputBufferId) Verify.verify(com.google.common.base.Verify.verify) Failures.checkCondition(io.prestosql.util.Failures.checkCondition) Objects.requireNonNull(java.util.Objects.requireNonNull) TimeStat(io.airlift.stats.TimeStat) REPLICATE(io.prestosql.sql.planner.plan.ExchangeNode.Type.REPLICATE) ExecutorService(java.util.concurrent.ExecutorService) Ints(com.google.common.primitives.Ints) DynamicFilterService(io.prestosql.dynamicfilter.DynamicFilterService) MoreFutures.tryGetFutureValue(io.airlift.concurrent.MoreFutures.tryGetFutureValue) SystemSessionProperties.getConcurrentLifespansPerNode(io.prestosql.SystemSessionProperties.getConcurrentLifespansPerNode) Collectors.toList(java.util.stream.Collectors.toList) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) BasicStageStats.aggregateBasicStageStats(io.prestosql.execution.BasicStageStats.aggregateBasicStageStats) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) StageExecutionPlan(io.prestosql.sql.planner.StageExecutionPlan) StageId(io.prestosql.execution.StageId) ArrayList(java.util.ArrayList) SqlStageExecution.createSqlStageExecution(io.prestosql.execution.SqlStageExecution.createSqlStageExecution) SqlStageExecution(io.prestosql.execution.SqlStageExecution) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) RemoteSourceNode(io.prestosql.sql.planner.plan.RemoteSourceNode) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) NodePartitionMap(io.prestosql.sql.planner.NodePartitionMap) Collection(java.util.Collection) CatalogName(io.prestosql.spi.connector.CatalogName) PartitioningHandle(io.prestosql.sql.planner.PartitioningHandle) SplitSource(io.prestosql.split.SplitSource) Map(java.util.Map) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) NodeTaskMap(io.prestosql.execution.NodeTaskMap) NodePartitionMap(io.prestosql.sql.planner.NodePartitionMap) ImmutableMap(com.google.common.collect.ImmutableMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap)

Example 3 with NodePartitioningManager

use of io.prestosql.sql.planner.NodePartitioningManager in project hetu-core by openlookeng.

the class ServerMainModule method setup.

@Override
protected void setup(Binder binder) {
    ServerConfig serverConfig = buildConfigObject(ServerConfig.class);
    if (serverConfig.isCoordinator()) {
        install(new CoordinatorModule());
    } else {
        install(new WorkerModule());
    }
    configBinder(binder).bindConfigDefaults(HttpServerConfig.class, httpServerConfig -> {
        httpServerConfig.setAdminEnabled(false);
    });
    install(new InternalCommunicationModule());
    configBinder(binder).bindConfig(FeaturesConfig.class);
    configBinder(binder).bindConfig(PasswordSecurityConfig.class);
    binder.bind(SqlParser.class).in(Scopes.SINGLETON);
    binder.bind(SqlParserOptions.class).toInstance(sqlParserOptions);
    sqlParserOptions.useEnhancedErrorHandler(serverConfig.isEnhancedErrorReporting());
    jaxrsBinder(binder).bind(ThrowableMapper.class);
    configBinder(binder).bindConfig(QueryManagerConfig.class);
    configBinder(binder).bindConfig(HetuConfig.class);
    configBinder(binder).bindConfig(SqlEnvironmentConfig.class);
    newOptionalBinder(binder, ExplainAnalyzeContext.class);
    // GC Monitor
    binder.bind(GcMonitor.class).to(JmxGcMonitor.class).in(Scopes.SINGLETON);
    // session properties
    binder.bind(SessionPropertyManager.class).in(Scopes.SINGLETON);
    binder.bind(SystemSessionProperties.class).in(Scopes.SINGLETON);
    binder.bind(SessionPropertyDefaults.class).in(Scopes.SINGLETON);
    // schema properties
    binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON);
    // table properties
    binder.bind(TablePropertyManager.class).in(Scopes.SINGLETON);
    // column properties
    binder.bind(ColumnPropertyManager.class).in(Scopes.SINGLETON);
    // analyze properties
    binder.bind(AnalyzePropertyManager.class).in(Scopes.SINGLETON);
    newSetBinder(binder, Filter.class, TheServlet.class).addBinding().to(HttpServerAvailableCheckFilter.class).in(Scopes.SINGLETON);
    // node manager
    discoveryBinder(binder).bindSelector("presto");
    binder.bind(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
    binder.bind(InternalNodeManager.class).to(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
    newExporter(binder).export(DiscoveryNodeManager.class).withGeneratedName();
    httpClientBinder(binder).bindHttpClient("node-manager", ForNodeManager.class).withTracing().withConfigDefaults(config -> {
        config.setIdleTimeout(serverConfig.getHttpClientIdleTimeout());
        config.setRequestTimeout(serverConfig.getHttpClientRequestTimeout());
    });
    // node scheduler
    // TODO: remove from NodePartitioningManager and move to CoordinatorModule
    configBinder(binder).bindConfig(NodeSchedulerConfig.class);
    binder.bind(NodeScheduler.class).in(Scopes.SINGLETON);
    binder.bind(NodeSchedulerExporter.class).in(Scopes.SINGLETON);
    binder.bind(NodeTaskMap.class).in(Scopes.SINGLETON);
    newExporter(binder).export(NodeScheduler.class).withGeneratedName();
    // network topology
    // TODO: move to CoordinatorModule when NodeScheduler is moved
    install(installModuleIf(NodeSchedulerConfig.class, config -> LEGACY.equalsIgnoreCase(config.getNetworkTopology()), moduleBinder -> moduleBinder.bind(NetworkTopology.class).to(LegacyNetworkTopology.class).in(Scopes.SINGLETON)));
    install(installModuleIf(NodeSchedulerConfig.class, config -> FLAT.equalsIgnoreCase(config.getNetworkTopology()), moduleBinder -> moduleBinder.bind(NetworkTopology.class).to(FlatNetworkTopology.class).in(Scopes.SINGLETON)));
    // task execution
    jaxrsBinder(binder).bind(TaskResource.class);
    newExporter(binder).export(TaskResource.class).withGeneratedName();
    jaxrsBinder(binder).bind(TaskExecutorResource.class);
    newExporter(binder).export(TaskExecutorResource.class).withGeneratedName();
    binder.bind(TaskManagementExecutor.class).in(Scopes.SINGLETON);
    binder.bind(SqlTaskManager.class).in(Scopes.SINGLETON);
    binder.bind(TaskManager.class).to(Key.get(SqlTaskManager.class));
    // memory revoking scheduler
    binder.bind(MemoryRevokingScheduler.class).in(Scopes.SINGLETON);
    // Add monitoring for JVM pauses
    binder.bind(PauseMeter.class).in(Scopes.SINGLETON);
    newExporter(binder).export(PauseMeter.class).withGeneratedName();
    configBinder(binder).bindConfig(MemoryManagerConfig.class);
    configBinder(binder).bindConfig(NodeMemoryConfig.class);
    binder.bind(LocalMemoryManager.class).in(Scopes.SINGLETON);
    binder.bind(LocalMemoryManagerExporter.class).in(Scopes.SINGLETON);
    binder.bind(EmbedVersion.class).in(Scopes.SINGLETON);
    newExporter(binder).export(TaskManager.class).withGeneratedName();
    binder.bind(TaskExecutor.class).in(Scopes.SINGLETON);
    newExporter(binder).export(TaskExecutor.class).withGeneratedName();
    binder.bind(MultilevelSplitQueue.class).in(Scopes.SINGLETON);
    newExporter(binder).export(MultilevelSplitQueue.class).withGeneratedName();
    binder.bind(LocalExecutionPlanner.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(CompilerConfig.class);
    binder.bind(ExpressionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ExpressionCompiler.class).withGeneratedName();
    binder.bind(PageFunctionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(PageFunctionCompiler.class).withGeneratedName();
    configBinder(binder).bindConfig(TaskManagerConfig.class);
    binder.bind(IndexJoinLookupStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(IndexJoinLookupStats.class).withGeneratedName();
    binder.bind(AsyncHttpExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(AsyncHttpExecutionMBean.class).withGeneratedName();
    binder.bind(JoinFilterFunctionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinFilterFunctionCompiler.class).withGeneratedName();
    binder.bind(JoinCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinCompiler.class).withGeneratedName();
    binder.bind(OrderingCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(OrderingCompiler.class).withGeneratedName();
    binder.bind(PagesIndex.Factory.class).to(PagesIndex.DefaultFactory.class);
    binder.bind(LookupJoinOperators.class).in(Scopes.SINGLETON);
    jsonCodecBinder(binder).bindJsonCodec(TaskStatus.class);
    jsonCodecBinder(binder).bindJsonCodec(StageInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class);
    smileCodecBinder(binder).bindSmileCodec(TaskStatus.class);
    smileCodecBinder(binder).bindSmileCodec(TaskInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(OperatorStats.class);
    jsonCodecBinder(binder).bindJsonCodec(ExecutionFailureInfo.class);
    jaxrsBinder(binder).bind(PagesResponseWriter.class);
    // exchange client
    binder.bind(ExchangeClientSupplier.class).to(ExchangeClientFactory.class).in(Scopes.SINGLETON);
    httpClientBinder(binder).bindHttpClient("exchange", ForExchange.class).withTracing().withFilter(GenerateTraceTokenRequestFilter.class).withConfigDefaults(config -> {
        config.setIdleTimeout(serverConfig.getHttpClientIdleTimeout());
        config.setRequestTimeout(serverConfig.getHttpClientRequestTimeout());
        config.setMaxConnectionsPerServer(250);
        config.setMaxContentLength(new DataSize(32, MEGABYTE));
    });
    configBinder(binder).bindConfig(ExchangeClientConfig.class);
    binder.bind(ExchangeExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ExchangeExecutionMBean.class).withGeneratedName();
    // execution
    binder.bind(LocationFactory.class).to(HttpLocationFactory.class).in(Scopes.SINGLETON);
    // memory manager
    jaxrsBinder(binder).bind(MemoryResource.class);
    jsonCodecBinder(binder).bindJsonCodec(MemoryInfo.class);
    jsonCodecBinder(binder).bindJsonCodec(MemoryPoolAssignmentsRequest.class);
    smileCodecBinder(binder).bindSmileCodec(MemoryInfo.class);
    smileCodecBinder(binder).bindSmileCodec(MemoryPoolAssignmentsRequest.class);
    // transaction manager
    configBinder(binder).bindConfig(TransactionManagerConfig.class);
    // data stream provider
    binder.bind(PageSourceManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSourceProvider.class).to(PageSourceManager.class).in(Scopes.SINGLETON);
    // page sink provider
    binder.bind(PageSinkManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSinkProvider.class).to(PageSinkManager.class).in(Scopes.SINGLETON);
    binder.bind(StaticCatalogStore.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(StaticCatalogStoreConfig.class);
    binder.bind(FunctionAndTypeManager.class).in(Scopes.SINGLETON);
    binder.bind(MetadataManager.class).in(Scopes.SINGLETON);
    binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON);
    binder.bind(DomainTranslator.class).to(RowExpressionDomainTranslator.class).in(Scopes.SINGLETON);
    binder.bind(DeterminismEvaluator.class).to(RowExpressionDeterminismEvaluator.class).in(Scopes.SINGLETON);
    // type
    binder.bind(TypeManager.class).to(FunctionAndTypeManager.class).in(Scopes.SINGLETON);
    binder.bind(TypeAnalyzer.class).in(Scopes.SINGLETON);
    jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
    newSetBinder(binder, Type.class);
    binder.bind(Kryo.class).in(Scopes.SINGLETON);
    // split manager
    binder.bind(SplitManager.class).in(Scopes.SINGLETON);
    // node partitioning manager
    binder.bind(NodePartitioningManager.class).in(Scopes.SINGLETON);
    // connector plan optimizer manager
    binder.bind(ConnectorPlanOptimizerManager.class).in(Scopes.SINGLETON);
    // index manager
    binder.bind(IndexManager.class).in(Scopes.SINGLETON);
    // handle resolver
    binder.install(new HandleJsonModule());
    // connector
    binder.bind(ScalarStatsCalculator.class).in(Scopes.SINGLETON);
    binder.bind(StatsNormalizer.class).in(Scopes.SINGLETON);
    binder.bind(FilterStatsCalculator.class).in(Scopes.SINGLETON);
    binder.bind(ConnectorManager.class).in(Scopes.SINGLETON);
    // binding the DataCenterConnectorManager class for loading the
    // DC connector sub catalogs
    binder.bind(DataCenterConnectorManager.class).in(Scopes.SINGLETON);
    // binding the DataCenterConnectorStore class for storing the
    // DC connector objects and their properties
    binder.bind(CatalogConnectorStore.class).in(Scopes.SINGLETON);
    // Cube manager
    binder.bind(CubeManager.class).in(Scopes.SINGLETON);
    // system connector
    binder.install(new SystemConnectorModule());
    // splits
    jsonCodecBinder(binder).bindJsonCodec(TaskUpdateRequest.class);
    jsonCodecBinder(binder).bindJsonCodec(ConnectorSplit.class);
    smileCodecBinder(binder).bindSmileCodec(TaskUpdateRequest.class);
    smileCodecBinder(binder).bindSmileCodec(ConnectorSplit.class);
    smileCodecBinder(binder).bindSmileCodec(PlanFragment.class);
    jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class);
    jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class);
    jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class);
    jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class);
    jsonBinder(binder).addDeserializerBinding(FunctionCall.class).to(FunctionCallDeserializer.class);
    // split monitor
    binder.bind(SplitMonitor.class).in(Scopes.SINGLETON);
    // Determine the NodeVersion
    NodeVersion nodeVersion = new NodeVersion(serverConfig.getPrestoVersion());
    binder.bind(NodeVersion.class).toInstance(nodeVersion);
    // presto announcement
    discoveryBinder(binder).bindHttpAnnouncement("presto").addProperty("node_version", nodeVersion.toString()).addProperty("coordinator", String.valueOf(serverConfig.isCoordinator())).addProperty("worker", String.valueOf(!serverConfig.isCoordinator() || buildConfigObject(NodeSchedulerConfig.class).isIncludeCoordinator()));
    // server info resource
    jaxrsBinder(binder).bind(ServerInfoResource.class);
    jsonCodecBinder(binder).bindJsonCodec(ServerInfo.class);
    // node status resource
    jaxrsBinder(binder).bind(StatusResource.class);
    jsonCodecBinder(binder).bindJsonCodec(NodeStatus.class);
    // dynamic catalog resource
    jsonCodecBinder(binder).bindJsonCodec(CatalogInfo.class);
    binder.bind(DynamicCatalogStore.class).in(Scopes.SINGLETON);
    binder.bind(DynamicCatalogScanner.class).in(Scopes.SINGLETON);
    binder.bind(CatalogStoreUtil.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(DynamicCatalogConfig.class);
    // plugin manager
    binder.bind(PluginManager.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(PluginManagerConfig.class);
    binder.bind(SeedStoreManager.class).in(Scopes.SINGLETON);
    binder.bind(FileSystemClientManager.class).in(Scopes.SINGLETON);
    binder.bind(CatalogManager.class).in(Scopes.SINGLETON);
    binder.bind(HetuMetaStoreManager.class).in(Scopes.SINGLETON);
    binder.bind(StaticFunctionNamespaceStore.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(StaticFunctionNamespaceStoreConfig.class);
    // block encodings
    jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class);
    jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class);
    // thread visualizer
    jaxrsBinder(binder).bind(ThreadResource.class);
    // PageSorter
    binder.bind(PageSorter.class).to(PagesIndexPageSorter.class).in(Scopes.SINGLETON);
    // PageIndexer
    binder.bind(PageIndexerFactory.class).to(GroupByHashPageIndexerFactory.class).in(Scopes.SINGLETON);
    // Finalizer
    binder.bind(FinalizerService.class).in(Scopes.SINGLETON);
    // HeuristicIndexerManager
    binder.bind(HeuristicIndexerManager.class).in(Scopes.SINGLETON);
    // SnapshotUtils
    binder.bind(SnapshotUtils.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(SnapshotConfig.class);
    // Spiller
    binder.bind(SpillerFactory.class).to(GenericSpillerFactory.class).in(Scopes.SINGLETON);
    binder.bind(SingleStreamSpillerFactory.class).to(FileSingleStreamSpillerFactory.class).in(Scopes.SINGLETON);
    binder.bind(PartitioningSpillerFactory.class).to(GenericPartitioningSpillerFactory.class).in(Scopes.SINGLETON);
    binder.bind(SpillerStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(SpillerFactory.class).withGeneratedName();
    binder.bind(LocalSpillManager.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(NodeSpillConfig.class);
    // cleanup
    binder.bind(ExecutorCleanup.class).in(Scopes.SINGLETON);
    // State store
    binder.bind(StateStoreProvider.class).to(LocalStateStoreProvider.class).in(Scopes.SINGLETON);
    // State store listener manager
    binder.bind(StateStoreListenerManager.class).in(Scopes.SINGLETON);
    // dynamic filter listener service
    binder.bind(DynamicFilterCacheManager.class).in(Scopes.SINGLETON);
}
Also used : TaskManagementExecutor(io.prestosql.execution.TaskManagementExecutor) FilterStatsCalculator(io.prestosql.cost.FilterStatsCalculator) ConnectorManager(io.prestosql.connector.ConnectorManager) FlatNetworkTopology(io.prestosql.execution.scheduler.FlatNetworkTopology) HttpServerConfig(io.airlift.http.server.HttpServerConfig) PauseMeter(io.airlift.stats.PauseMeter) DeterminismEvaluator(io.prestosql.spi.relation.DeterminismEvaluator) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) SnapshotConfig(io.prestosql.snapshot.SnapshotConfig) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) SqlParserOptions(io.prestosql.sql.parser.SqlParserOptions) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) SingleStreamSpillerFactory(io.prestosql.spiller.SingleStreamSpillerFactory) Multibinder.newSetBinder(com.google.inject.multibindings.Multibinder.newSetBinder) Filter(javax.servlet.Filter) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) ExchangeClientFactory(io.prestosql.operator.ExchangeClientFactory) TransactionManagerConfig(io.prestosql.transaction.TransactionManagerConfig) PageIndexerFactory(io.prestosql.spi.PageIndexerFactory) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) JsonCodecBinder.jsonCodecBinder(io.airlift.json.JsonCodecBinder.jsonCodecBinder) JmxGcMonitor(io.airlift.stats.JmxGcMonitor) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) SqlTaskManager(io.prestosql.execution.SqlTaskManager) TaskStatus(io.prestosql.execution.TaskStatus) DomainTranslator(io.prestosql.spi.relation.DomainTranslator) SchemaPropertyManager(io.prestosql.metadata.SchemaPropertyManager) ExplainAnalyzeContext(io.prestosql.execution.ExplainAnalyzeContext) NodeSchedulerExporter(io.prestosql.execution.scheduler.NodeSchedulerExporter) FileSingleStreamSpillerFactory(io.prestosql.spiller.FileSingleStreamSpillerFactory) DynamicCatalogStore(io.prestosql.catalog.DynamicCatalogStore) BlockJsonSerde(io.prestosql.block.BlockJsonSerde) LocationFactory(io.prestosql.execution.LocationFactory) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) Metadata(io.prestosql.metadata.Metadata) MetadataManager(io.prestosql.metadata.MetadataManager) HttpClientBinder.httpClientBinder(io.airlift.http.client.HttpClientBinder.httpClientBinder) PageSinkManager(io.prestosql.split.PageSinkManager) PasswordSecurityConfig(io.prestosql.security.PasswordSecurityConfig) DiscoveryBinder.discoveryBinder(io.airlift.discovery.client.DiscoveryBinder.discoveryBinder) MemoryRevokingScheduler(io.prestosql.execution.MemoryRevokingScheduler) ConfigBinder.configBinder(io.airlift.configuration.ConfigBinder.configBinder) StaticFunctionNamespaceStore(io.prestosql.metadata.StaticFunctionNamespaceStore) NodeMemoryConfig(io.prestosql.memory.NodeMemoryConfig) OperatorStats(io.prestosql.operator.OperatorStats) TaskManagerConfig(io.prestosql.execution.TaskManagerConfig) Slice(io.airlift.slice.Slice) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) EmbedVersion(io.prestosql.version.EmbedVersion) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) ForExchange(io.prestosql.operator.ForExchange) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) BlockEncodingSerde(io.prestosql.spi.block.BlockEncodingSerde) ServerInfo(io.prestosql.client.ServerInfo) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) CatalogConnectorStore(io.prestosql.connector.CatalogConnectorStore) SystemConnectorModule(io.prestosql.connector.system.SystemConnectorModule) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) Binder(com.google.inject.Binder) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) PageSourceManager(io.prestosql.split.PageSourceManager) NodeTaskMap(io.prestosql.execution.NodeTaskMap) TaskManager(io.prestosql.execution.TaskManager) SpillerFactory(io.prestosql.spiller.SpillerFactory) InternalNodeManager(io.prestosql.metadata.InternalNodeManager) DiscoveryNodeManager(io.prestosql.metadata.DiscoveryNodeManager) ForNodeManager(io.prestosql.metadata.ForNodeManager) FLAT(io.prestosql.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.FLAT) NetworkTopology(io.prestosql.execution.scheduler.NetworkTopology) TypeManager(io.prestosql.spi.type.TypeManager) MultilevelSplitQueue(io.prestosql.execution.executor.MultilevelSplitQueue) FinalizerService(io.prestosql.util.FinalizerService) CubeManager(io.prestosql.cube.CubeManager) MemoryPoolAssignmentsRequest(io.prestosql.memory.MemoryPoolAssignmentsRequest) SmileCodecBinder.smileCodecBinder(io.prestosql.protocol.SmileCodecBinder.smileCodecBinder) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) ExchangeClientSupplier(io.prestosql.operator.ExchangeClientSupplier) SplitManager(io.prestosql.split.SplitManager) CatalogManager(io.prestosql.metadata.CatalogManager) LocalMemoryManagerExporter(io.prestosql.memory.LocalMemoryManagerExporter) GcMonitor(io.airlift.stats.GcMonitor) Expression(io.prestosql.sql.tree.Expression) TypeDeserializer(io.prestosql.type.TypeDeserializer) MemoryInfo(io.prestosql.memory.MemoryInfo) TablePropertyManager(io.prestosql.metadata.TablePropertyManager) TheServlet(io.airlift.http.server.TheServlet) SystemSessionProperties(io.prestosql.SystemSessionProperties) LocalMemoryManager(io.prestosql.memory.LocalMemoryManager) Key(com.google.inject.Key) DataCenterConnectorManager(io.prestosql.connector.DataCenterConnectorManager) SqlParser(io.prestosql.sql.parser.SqlParser) DynamicCatalogConfig(io.prestosql.catalog.DynamicCatalogConfig) StatsNormalizer(io.prestosql.cost.StatsNormalizer) QueryManagerConfig(io.prestosql.execution.QueryManagerConfig) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) JsonBinder.jsonBinder(io.airlift.json.JsonBinder.jsonBinder) Kryo(com.esotericsoftware.kryo.Kryo) PageSinkProvider(io.prestosql.split.PageSinkProvider) HetuConfig(io.prestosql.utils.HetuConfig) PreDestroy(javax.annotation.PreDestroy) ColumnPropertyManager(io.prestosql.metadata.ColumnPropertyManager) ConnectorPlanOptimizerManager(io.prestosql.sql.planner.ConnectorPlanOptimizerManager) ExchangeClientConfig(io.prestosql.operator.ExchangeClientConfig) ExportBinder.newExporter(org.weakref.jmx.guice.ExportBinder.newExporter) SnapshotUtils(io.prestosql.snapshot.SnapshotUtils) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Type(io.prestosql.spi.type.Type) ExpressionSerializer(io.prestosql.sql.Serialization.ExpressionSerializer) OptionalBinder.newOptionalBinder(com.google.inject.multibindings.OptionalBinder.newOptionalBinder) PartitioningSpillerFactory(io.prestosql.spiller.PartitioningSpillerFactory) ConditionalModule.installModuleIf(io.airlift.configuration.ConditionalModule.installModuleIf) TaskExecutor(io.prestosql.execution.executor.TaskExecutor) TaskInfo(io.prestosql.execution.TaskInfo) GenericPartitioningSpillerFactory(io.prestosql.spiller.GenericPartitioningSpillerFactory) PageSourceProvider(io.prestosql.split.PageSourceProvider) DataSize(io.airlift.units.DataSize) List(java.util.List) SessionPropertyManager(io.prestosql.metadata.SessionPropertyManager) StageInfo(io.prestosql.execution.StageInfo) JaxrsBinder.jaxrsBinder(io.airlift.jaxrs.JaxrsBinder.jaxrsBinder) NodeSpillConfig(io.prestosql.spiller.NodeSpillConfig) CatalogStoreUtil(io.prestosql.catalog.CatalogStoreUtil) NodeVersion(io.prestosql.client.NodeVersion) FunctionAndTypeManager(io.prestosql.metadata.FunctionAndTypeManager) StaticFunctionNamespaceStoreConfig(io.prestosql.metadata.StaticFunctionNamespaceStoreConfig) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) MemoryManagerConfig(io.prestosql.memory.MemoryManagerConfig) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) SqlEnvironmentConfig(io.prestosql.sql.SqlEnvironmentConfig) Singleton(javax.inject.Singleton) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) Inject(javax.inject.Inject) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) FunctionCallDeserializer(io.prestosql.sql.Serialization.FunctionCallDeserializer) AbstractConfigurationAwareModule(io.airlift.configuration.AbstractConfigurationAwareModule) ImmutableList(com.google.common.collect.ImmutableList) FunctionCall(io.prestosql.sql.tree.FunctionCall) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) Objects.requireNonNull(java.util.Objects.requireNonNull) StaticCatalogStore(io.prestosql.metadata.StaticCatalogStore) LEGACY(io.prestosql.execution.scheduler.NodeSchedulerConfig.NetworkTopologyType.LEGACY) PagesIndexPageSorter(io.prestosql.PagesIndexPageSorter) LocalSpillManager(io.prestosql.spiller.LocalSpillManager) PagesIndex(io.prestosql.operator.PagesIndex) Block(io.prestosql.spi.block.Block) ExecutorService(java.util.concurrent.ExecutorService) PlanFragment(io.prestosql.sql.planner.PlanFragment) ScalarStatsCalculator(io.prestosql.cost.ScalarStatsCalculator) IndexManager(io.prestosql.index.IndexManager) SplitMonitor(io.prestosql.event.SplitMonitor) CompilerConfig(io.prestosql.sql.planner.CompilerConfig) AnalyzePropertyManager(io.prestosql.metadata.AnalyzePropertyManager) StaticCatalogStoreConfig(io.prestosql.metadata.StaticCatalogStoreConfig) PageSorter(io.prestosql.spi.PageSorter) Scopes(com.google.inject.Scopes) MemoryResource(io.prestosql.memory.MemoryResource) HttpLocationFactory(io.prestosql.server.remotetask.HttpLocationFactory) Provides(com.google.inject.Provides) ExpressionDeserializer(io.prestosql.sql.Serialization.ExpressionDeserializer) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) DynamicCatalogScanner(io.prestosql.catalog.DynamicCatalogScanner) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) SpillerStats(io.prestosql.spiller.SpillerStats) ExecutionFailureInfo(io.prestosql.execution.ExecutionFailureInfo) CatalogInfo(io.prestosql.catalog.CatalogInfo) HandleJsonModule(io.prestosql.metadata.HandleJsonModule) DataCenterConnectorManager(io.prestosql.connector.DataCenterConnectorManager) NodeVersion(io.prestosql.client.NodeVersion) FunctionAndTypeManager(io.prestosql.metadata.FunctionAndTypeManager) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) GenericPartitioningSpillerFactory(io.prestosql.spiller.GenericPartitioningSpillerFactory) StaticFunctionNamespaceStore(io.prestosql.metadata.StaticFunctionNamespaceStore) BlockJsonSerde(io.prestosql.block.BlockJsonSerde) MultilevelSplitQueue(io.prestosql.execution.executor.MultilevelSplitQueue) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) ScalarStatsCalculator(io.prestosql.cost.ScalarStatsCalculator) DynamicCatalogStore(io.prestosql.catalog.DynamicCatalogStore) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) PagesIndex(io.prestosql.operator.PagesIndex) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) LocalMemoryManager(io.prestosql.memory.LocalMemoryManager) SnapshotUtils(io.prestosql.snapshot.SnapshotUtils) SystemConnectorModule(io.prestosql.connector.system.SystemConnectorModule) FunctionCall(io.prestosql.sql.tree.FunctionCall) EmbedVersion(io.prestosql.version.EmbedVersion) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) PageSinkManager(io.prestosql.split.PageSinkManager) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) ConnectorPlanOptimizerManager(io.prestosql.sql.planner.ConnectorPlanOptimizerManager) StatsNormalizer(io.prestosql.cost.StatsNormalizer) FlatNetworkTopology(io.prestosql.execution.scheduler.FlatNetworkTopology) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) NodeSchedulerExporter(io.prestosql.execution.scheduler.NodeSchedulerExporter) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) ConnectorManager(io.prestosql.connector.ConnectorManager) DataCenterConnectorManager(io.prestosql.connector.DataCenterConnectorManager) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) CatalogManager(io.prestosql.metadata.CatalogManager) Type(io.prestosql.spi.type.Type) DynamicCatalogScanner(io.prestosql.catalog.DynamicCatalogScanner) Expression(io.prestosql.sql.tree.Expression) FinalizerService(io.prestosql.util.FinalizerService) TaskManagementExecutor(io.prestosql.execution.TaskManagementExecutor) MemoryRevokingScheduler(io.prestosql.execution.MemoryRevokingScheduler) TablePropertyManager(io.prestosql.metadata.TablePropertyManager) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) TheServlet(io.airlift.http.server.TheServlet) PageSourceManager(io.prestosql.split.PageSourceManager) HttpServerConfig(io.airlift.http.server.HttpServerConfig) PagesIndexPageSorter(io.prestosql.PagesIndexPageSorter) ExchangeClientFactory(io.prestosql.operator.ExchangeClientFactory) FileSingleStreamSpillerFactory(io.prestosql.spiller.FileSingleStreamSpillerFactory) DataSize(io.airlift.units.DataSize) SessionPropertyManager(io.prestosql.metadata.SessionPropertyManager) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) SqlParserOptions(io.prestosql.sql.parser.SqlParserOptions) ForExchange(io.prestosql.operator.ForExchange) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) StaticCatalogStore(io.prestosql.metadata.StaticCatalogStore) NodeTaskMap(io.prestosql.execution.NodeTaskMap) SplitMonitor(io.prestosql.event.SplitMonitor) HttpLocationFactory(io.prestosql.server.remotetask.HttpLocationFactory) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) SqlParser(io.prestosql.sql.parser.SqlParser) PauseMeter(io.airlift.stats.PauseMeter) SpillerStats(io.prestosql.spiller.SpillerStats) IndexManager(io.prestosql.index.IndexManager) LocalSpillManager(io.prestosql.spiller.LocalSpillManager) SqlTaskManager(io.prestosql.execution.SqlTaskManager) TaskExecutor(io.prestosql.execution.executor.TaskExecutor) Slice(io.airlift.slice.Slice) Block(io.prestosql.spi.block.Block) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory) ColumnPropertyManager(io.prestosql.metadata.ColumnPropertyManager) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) SingleStreamSpillerFactory(io.prestosql.spiller.SingleStreamSpillerFactory) ExchangeClientFactory(io.prestosql.operator.ExchangeClientFactory) PageIndexerFactory(io.prestosql.spi.PageIndexerFactory) FileSingleStreamSpillerFactory(io.prestosql.spiller.FileSingleStreamSpillerFactory) LocationFactory(io.prestosql.execution.LocationFactory) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory) SpillerFactory(io.prestosql.spiller.SpillerFactory) PartitioningSpillerFactory(io.prestosql.spiller.PartitioningSpillerFactory) GenericPartitioningSpillerFactory(io.prestosql.spiller.GenericPartitioningSpillerFactory) HttpLocationFactory(io.prestosql.server.remotetask.HttpLocationFactory) LocalMemoryManagerExporter(io.prestosql.memory.LocalMemoryManagerExporter) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) HandleJsonModule(io.prestosql.metadata.HandleJsonModule) SchemaPropertyManager(io.prestosql.metadata.SchemaPropertyManager) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) AnalyzePropertyManager(io.prestosql.metadata.AnalyzePropertyManager) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) CatalogStoreUtil(io.prestosql.catalog.CatalogStoreUtil) FilterStatsCalculator(io.prestosql.cost.FilterStatsCalculator) SingleStreamSpillerFactory(io.prestosql.spiller.SingleStreamSpillerFactory) FileSingleStreamSpillerFactory(io.prestosql.spiller.FileSingleStreamSpillerFactory) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) SpillerFactory(io.prestosql.spiller.SpillerFactory) PartitioningSpillerFactory(io.prestosql.spiller.PartitioningSpillerFactory) GenericPartitioningSpillerFactory(io.prestosql.spiller.GenericPartitioningSpillerFactory) SplitManager(io.prestosql.split.SplitManager) CubeManager(io.prestosql.cube.CubeManager) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) SqlTaskManager(io.prestosql.execution.SqlTaskManager) TaskManager(io.prestosql.execution.TaskManager) MetadataManager(io.prestosql.metadata.MetadataManager) Filter(javax.servlet.Filter) CatalogConnectorStore(io.prestosql.connector.CatalogConnectorStore) JmxGcMonitor(io.airlift.stats.JmxGcMonitor) DiscoveryNodeManager(io.prestosql.metadata.DiscoveryNodeManager) Kryo(com.esotericsoftware.kryo.Kryo) SystemSessionProperties(io.prestosql.SystemSessionProperties)

Aggregations

ImmutableList (com.google.common.collect.ImmutableList)3 HeuristicIndexerManager (io.prestosql.heuristicindex.HeuristicIndexerManager)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 SystemSessionProperties (io.prestosql.SystemSessionProperties)2 LocationFactory (io.prestosql.execution.LocationFactory)2 NodeTaskMap (io.prestosql.execution.NodeTaskMap)2 StageInfo (io.prestosql.execution.StageInfo)2 TaskStatus (io.prestosql.execution.TaskStatus)2 NodePartitioningManager (io.prestosql.sql.planner.NodePartitioningManager)2 List (java.util.List)2 Objects.requireNonNull (java.util.Objects.requireNonNull)2 ExecutorService (java.util.concurrent.ExecutorService)2 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)2 Kryo (com.esotericsoftware.kryo.Kryo)1 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 Preconditions.checkState (com.google.common.base.Preconditions.checkState)1 Verify.verify (com.google.common.base.Verify.verify)1 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)1 ImmutableMap.toImmutableMap (com.google.common.collect.ImmutableMap.toImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1