Search in sources :

Example 1 with NodeScheduler

use of io.trino.execution.scheduler.NodeScheduler in project trino by trinodb.

the class ServerMainModule method setup.

@Override
protected void setup(Binder binder) {
    ServerConfig serverConfig = buildConfigObject(ServerConfig.class);
    if (serverConfig.isCoordinator()) {
        install(new CoordinatorModule());
    } else {
        install(new WorkerModule());
    }
    binder.bind(StartupStatus.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfigDefaults(HttpServerConfig.class, httpServerConfig -> {
        httpServerConfig.setAdminEnabled(false);
    });
    binder.bind(PreparedStatementEncoder.class).in(Scopes.SINGLETON);
    binder.bind(HttpRequestSessionContextFactory.class).in(Scopes.SINGLETON);
    install(new InternalCommunicationModule());
    configBinder(binder).bindConfig(FeaturesConfig.class);
    configBinder(binder).bindConfig(OptimizerConfig.class);
    configBinder(binder).bindConfig(ProtocolConfig.class);
    binder.bind(SqlParser.class).in(Scopes.SINGLETON);
    jaxrsBinder(binder).bind(ThrowableMapper.class);
    configBinder(binder).bindConfig(QueryManagerConfig.class);
    configBinder(binder).bindConfig(SqlEnvironmentConfig.class);
    newOptionalBinder(binder, ExplainAnalyzeContext.class);
    binder.bind(StatementAnalyzerFactory.class).in(Scopes.SINGLETON);
    // GC Monitor
    binder.bind(GcMonitor.class).to(JmxGcMonitor.class).in(Scopes.SINGLETON);
    newExporter(binder).export(GcMonitor.class).withGeneratedName();
    // session properties
    newSetBinder(binder, SystemSessionPropertiesProvider.class).addBinding().to(SystemSessionProperties.class);
    binder.bind(SessionPropertyManager.class).in(Scopes.SINGLETON);
    binder.bind(SystemSessionProperties.class).in(Scopes.SINGLETON);
    binder.bind(SessionPropertyDefaults.class).in(Scopes.SINGLETON);
    // schema properties
    binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON);
    // table properties
    binder.bind(TablePropertyManager.class).in(Scopes.SINGLETON);
    // materialized view properties
    binder.bind(MaterializedViewPropertyManager.class).in(Scopes.SINGLETON);
    // column properties
    binder.bind(ColumnPropertyManager.class).in(Scopes.SINGLETON);
    // analyze properties
    binder.bind(AnalyzePropertyManager.class).in(Scopes.SINGLETON);
    // table procedures properties
    binder.bind(TableProceduresPropertyManager.class).in(Scopes.SINGLETON);
    // node manager
    discoveryBinder(binder).bindSelector("trino");
    binder.bind(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
    binder.bind(InternalNodeManager.class).to(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
    newExporter(binder).export(DiscoveryNodeManager.class).withGeneratedName();
    httpClientBinder(binder).bindHttpClient("node-manager", ForNodeManager.class).withTracing().withConfigDefaults(config -> {
        config.setIdleTimeout(new Duration(30, SECONDS));
        config.setRequestTimeout(new Duration(10, SECONDS));
    });
    // node scheduler
    // TODO: remove from NodePartitioningManager and move to CoordinatorModule
    configBinder(binder).bindConfig(NodeSchedulerConfig.class);
    binder.bind(NodeScheduler.class).in(Scopes.SINGLETON);
    binder.bind(NodeTaskMap.class).in(Scopes.SINGLETON);
    newExporter(binder).export(NodeScheduler.class).withGeneratedName();
    // network topology
    // TODO: move to CoordinatorModule when NodeScheduler is moved
    install(conditionalModule(NodeSchedulerConfig.class, config -> UNIFORM == config.getNodeSchedulerPolicy(), new UniformNodeSelectorModule()));
    install(conditionalModule(NodeSchedulerConfig.class, config -> TOPOLOGY == config.getNodeSchedulerPolicy(), new TopologyAwareNodeSelectorModule()));
    // task execution
    configBinder(binder).bindConfig(FailureInjectionConfig.class);
    binder.bind(FailureInjector.class).in(Scopes.SINGLETON);
    jaxrsBinder(binder).bind(TaskResource.class);
    newExporter(binder).export(TaskResource.class).withGeneratedName();
    jaxrsBinder(binder).bind(TaskExecutorResource.class);
    newExporter(binder).export(TaskExecutorResource.class).withGeneratedName();
    binder.bind(TaskManagementExecutor.class).in(Scopes.SINGLETON);
    binder.bind(SqlTaskManager.class).in(Scopes.SINGLETON);
    binder.bind(TableExecuteContextManager.class).in(Scopes.SINGLETON);
    // memory revoking scheduler
    binder.bind(MemoryRevokingScheduler.class).in(Scopes.SINGLETON);
    // Add monitoring for JVM pauses
    binder.bind(PauseMeter.class).in(Scopes.SINGLETON);
    newExporter(binder).export(PauseMeter.class).withGeneratedName();
    configBinder(binder).bindConfig(MemoryManagerConfig.class);
    configBinder(binder).bindConfig(NodeMemoryConfig.class);
    binder.bind(LocalMemoryManager.class).in(Scopes.SINGLETON);
    binder.bind(LocalMemoryManagerExporter.class).in(Scopes.SINGLETON);
    newOptionalBinder(binder, VersionEmbedder.class).setDefault().to(EmbedVersion.class).in(Scopes.SINGLETON);
    newExporter(binder).export(SqlTaskManager.class).withGeneratedName();
    binder.bind(TaskExecutor.class).in(Scopes.SINGLETON);
    newExporter(binder).export(TaskExecutor.class).withGeneratedName();
    binder.bind(MultilevelSplitQueue.class).in(Scopes.SINGLETON);
    newExporter(binder).export(MultilevelSplitQueue.class).withGeneratedName();
    binder.bind(LocalExecutionPlanner.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(CompilerConfig.class);
    binder.bind(ExpressionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ExpressionCompiler.class).withGeneratedName();
    binder.bind(PageFunctionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(PageFunctionCompiler.class).withGeneratedName();
    configBinder(binder).bindConfig(TaskManagerConfig.class);
    binder.bind(IndexJoinLookupStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(IndexJoinLookupStats.class).withGeneratedName();
    binder.bind(AsyncHttpExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(AsyncHttpExecutionMBean.class).withGeneratedName();
    binder.bind(JoinFilterFunctionCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinFilterFunctionCompiler.class).withGeneratedName();
    binder.bind(JoinCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(JoinCompiler.class).withGeneratedName();
    binder.bind(OrderingCompiler.class).in(Scopes.SINGLETON);
    newExporter(binder).export(OrderingCompiler.class).withGeneratedName();
    binder.bind(PagesIndex.Factory.class).to(PagesIndex.DefaultFactory.class);
    newOptionalBinder(binder, OperatorFactories.class).setDefault().to(TrinoOperatorFactories.class).in(Scopes.SINGLETON);
    jaxrsBinder(binder).bind(PagesResponseWriter.class);
    // exchange client
    binder.bind(DirectExchangeClientSupplier.class).to(DirectExchangeClientFactory.class).in(Scopes.SINGLETON);
    httpClientBinder(binder).bindHttpClient("exchange", ForExchange.class).withTracing().withFilter(GenerateTraceTokenRequestFilter.class).withConfigDefaults(config -> {
        config.setIdleTimeout(new Duration(30, SECONDS));
        config.setRequestTimeout(new Duration(10, SECONDS));
        config.setMaxConnectionsPerServer(250);
        config.setMaxContentLength(DataSize.of(32, MEGABYTE));
    });
    configBinder(binder).bindConfig(DirectExchangeClientConfig.class);
    binder.bind(ExchangeExecutionMBean.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ExchangeExecutionMBean.class).withGeneratedName();
    // execution
    binder.bind(LocationFactory.class).to(HttpLocationFactory.class).in(Scopes.SINGLETON);
    // memory manager
    jaxrsBinder(binder).bind(MemoryResource.class);
    jsonCodecBinder(binder).bindJsonCodec(MemoryInfo.class);
    // transaction manager
    configBinder(binder).bindConfig(TransactionManagerConfig.class);
    // data stream provider
    binder.bind(PageSourceManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSourceProvider.class).to(PageSourceManager.class).in(Scopes.SINGLETON);
    // page sink provider
    binder.bind(PageSinkManager.class).in(Scopes.SINGLETON);
    binder.bind(PageSinkProvider.class).to(PageSinkManager.class).in(Scopes.SINGLETON);
    // metadata
    binder.bind(StaticCatalogStore.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(StaticCatalogStoreConfig.class);
    binder.bind(MetadataManager.class).in(Scopes.SINGLETON);
    binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON);
    newOptionalBinder(binder, SystemSecurityMetadata.class).setDefault().to(DisabledSystemSecurityMetadata.class).in(Scopes.SINGLETON);
    binder.bind(GlobalFunctionCatalog.class).in(Scopes.SINGLETON);
    binder.bind(TypeOperatorsCache.class).in(Scopes.SINGLETON);
    newExporter(binder).export(TypeOperatorsCache.class).as(factory -> factory.generatedNameOf(TypeOperators.class));
    binder.bind(BlockTypeOperators.class).in(Scopes.SINGLETON);
    newExporter(binder).export(TypeOperatorsCache.class).withGeneratedName();
    binder.bind(ProcedureRegistry.class).in(Scopes.SINGLETON);
    binder.bind(TableProceduresRegistry.class).in(Scopes.SINGLETON);
    binder.bind(PlannerContext.class).in(Scopes.SINGLETON);
    // function
    binder.bind(FunctionManager.class).in(Scopes.SINGLETON);
    newSetBinder(binder, FunctionBundle.class);
    binder.bind(RegisterFunctionBundles.class).asEagerSingleton();
    // type
    binder.bind(TypeAnalyzer.class).in(Scopes.SINGLETON);
    jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
    jsonBinder(binder).addDeserializerBinding(TypeSignature.class).to(TypeSignatureDeserializer.class);
    jsonBinder(binder).addKeyDeserializerBinding(TypeSignature.class).to(TypeSignatureKeyDeserializer.class);
    binder.bind(TypeRegistry.class).in(Scopes.SINGLETON);
    binder.bind(TypeManager.class).to(InternalTypeManager.class).in(Scopes.SINGLETON);
    newSetBinder(binder, Type.class);
    // split manager
    binder.bind(SplitManager.class).in(Scopes.SINGLETON);
    // node partitioning manager
    binder.bind(NodePartitioningManager.class).in(Scopes.SINGLETON);
    // index manager
    binder.bind(IndexManager.class).in(Scopes.SINGLETON);
    // handle resolver
    binder.install(new HandleJsonModule());
    // connector
    binder.bind(ConnectorManager.class).in(Scopes.SINGLETON);
    // system connector
    binder.install(new SystemConnectorModule());
    // slice
    jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class);
    jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class);
    // expression
    jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class);
    jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class);
    // split monitor
    binder.bind(SplitMonitor.class).in(Scopes.SINGLETON);
    // version and announcement
    binder.bind(NodeVersion.class).toInstance(new NodeVersion(nodeVersion));
    discoveryBinder(binder).bindHttpAnnouncement("trino").addProperty("node_version", nodeVersion).addProperty("coordinator", String.valueOf(serverConfig.isCoordinator()));
    // server info resource
    jaxrsBinder(binder).bind(ServerInfoResource.class);
    // node status resource
    jaxrsBinder(binder).bind(StatusResource.class);
    // plugin manager
    binder.bind(PluginManager.class).in(Scopes.SINGLETON);
    newOptionalBinder(binder, PluginsProvider.class).setDefault().to(ServerPluginsProvider.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(ServerPluginsProviderConfig.class);
    binder.bind(CatalogManager.class).in(Scopes.SINGLETON);
    // block encodings
    binder.bind(BlockEncodingManager.class).in(Scopes.SINGLETON);
    jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class);
    jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class);
    binder.bind(BlockEncodingSerde.class).to(InternalBlockEncodingSerde.class).in(Scopes.SINGLETON);
    // thread visualizer
    jaxrsBinder(binder).bind(ThreadResource.class);
    // PageSorter
    binder.bind(PageSorter.class).to(PagesIndexPageSorter.class).in(Scopes.SINGLETON);
    // PageIndexer
    binder.bind(PageIndexerFactory.class).to(GroupByHashPageIndexerFactory.class).in(Scopes.SINGLETON);
    // Finalizer
    binder.bind(FinalizerService.class).in(Scopes.SINGLETON);
    // Spiller
    binder.bind(SpillerFactory.class).to(GenericSpillerFactory.class).in(Scopes.SINGLETON);
    binder.bind(SingleStreamSpillerFactory.class).to(FileSingleStreamSpillerFactory.class).in(Scopes.SINGLETON);
    binder.bind(PartitioningSpillerFactory.class).to(GenericPartitioningSpillerFactory.class).in(Scopes.SINGLETON);
    binder.bind(SpillerStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(SpillerFactory.class).withGeneratedName();
    binder.bind(LocalSpillManager.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(NodeSpillConfig.class);
    // Dynamic Filtering
    configBinder(binder).bindConfig(DynamicFilterConfig.class);
    // dispatcher
    // TODO remove dispatcher fromm ServerMainModule, and bind dependent components only on coordinators
    newOptionalBinder(binder, DispatchManager.class);
    // Added for RuleStatsSystemTable
    // TODO: remove this when system tables are bound separately for coordinator and worker
    newOptionalBinder(binder, RuleStatsRecorder.class);
    // cleanup
    binder.bind(ExecutorCleanup.class).in(Scopes.SINGLETON);
}
Also used : TableProceduresPropertyManager(io.trino.metadata.TableProceduresPropertyManager) ConnectorManager(io.trino.connector.ConnectorManager) PagesIndexPageSorter(io.trino.operator.PagesIndexPageSorter) TopologyAwareNodeSelectorModule(io.trino.execution.scheduler.TopologyAwareNodeSelectorModule) ExpressionSerializer(io.trino.server.ExpressionSerialization.ExpressionSerializer) TypeOperators(io.trino.spi.type.TypeOperators) HttpServerConfig(io.airlift.http.server.HttpServerConfig) PauseMeter(io.airlift.stats.PauseMeter) MemoryRevokingScheduler(io.trino.execution.MemoryRevokingScheduler) TransactionManagerConfig(io.trino.transaction.TransactionManagerConfig) LocalSpillManager(io.trino.spiller.LocalSpillManager) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) FinalizerService(io.trino.util.FinalizerService) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) Multibinder.newSetBinder(com.google.inject.multibindings.Multibinder.newSetBinder) MetadataManager(io.trino.metadata.MetadataManager) FailureInjectionConfig(io.trino.execution.FailureInjectionConfig) StaticCatalogStore(io.trino.metadata.StaticCatalogStore) JsonCodecBinder.jsonCodecBinder(io.airlift.json.JsonCodecBinder.jsonCodecBinder) JmxGcMonitor(io.airlift.stats.JmxGcMonitor) LocalMemoryManagerExporter(io.trino.memory.LocalMemoryManagerExporter) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) PageSinkManager(io.trino.split.PageSinkManager) Set(java.util.Set) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) SystemSessionPropertiesProvider(io.trino.SystemSessionPropertiesProvider) PreparedStatementEncoder(io.trino.server.protocol.PreparedStatementEncoder) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) SystemConnectorModule(io.trino.connector.system.SystemConnectorModule) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) HttpClientBinder.httpClientBinder(io.airlift.http.client.HttpClientBinder.httpClientBinder) TaskManagerConfig(io.trino.execution.TaskManagerConfig) InternalBlockEncodingSerde(io.trino.metadata.InternalBlockEncodingSerde) DiscoveryBinder.discoveryBinder(io.airlift.discovery.client.DiscoveryBinder.discoveryBinder) VersionEmbedder(io.trino.spi.VersionEmbedder) EmbedVersion(io.trino.version.EmbedVersion) SpillerFactory(io.trino.spiller.SpillerFactory) ConfigBinder.configBinder(io.airlift.configuration.ConfigBinder.configBinder) ExplainAnalyzeContext(io.trino.execution.ExplainAnalyzeContext) InternalNodeManager(io.trino.metadata.InternalNodeManager) Slice(io.airlift.slice.Slice) SqlTaskManager(io.trino.execution.SqlTaskManager) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) PluginsProvider(io.trino.server.PluginManager.PluginsProvider) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) JoinCompiler(io.trino.sql.gen.JoinCompiler) MemoryInfo(io.trino.memory.MemoryInfo) MemoryResource(io.trino.memory.MemoryResource) MultilevelSplitQueue(io.trino.execution.executor.MultilevelSplitQueue) GroupByHashPageIndexerFactory(io.trino.operator.GroupByHashPageIndexerFactory) TypeRegistry(io.trino.metadata.TypeRegistry) RuleStatsRecorder(io.trino.sql.planner.RuleStatsRecorder) SliceDeserializer(io.trino.server.SliceSerialization.SliceDeserializer) Binder(com.google.inject.Binder) BlockJsonSerde(io.trino.block.BlockJsonSerde) SystemFunctionBundle(io.trino.metadata.SystemFunctionBundle) PageIndexerFactory(io.trino.spi.PageIndexerFactory) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) NodeVersion(io.trino.client.NodeVersion) TaskExecutor(io.trino.execution.executor.TaskExecutor) SqlEnvironmentConfig(io.trino.sql.SqlEnvironmentConfig) BlockTypeOperators(io.trino.type.BlockTypeOperators) Executor(java.util.concurrent.Executor) PageSinkProvider(io.trino.split.PageSinkProvider) SpillerStats(io.trino.spiller.SpillerStats) DirectExchangeClientConfig(io.trino.operator.DirectExchangeClientConfig) HttpLocationFactory(io.trino.server.remotetask.HttpLocationFactory) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) FunctionBundle(io.trino.metadata.FunctionBundle) TypeSignatureKeyDeserializer(io.trino.type.TypeSignatureKeyDeserializer) TypeAnalyzer(io.trino.sql.planner.TypeAnalyzer) InternalFunctionBundle(io.trino.metadata.InternalFunctionBundle) StatementAnalyzerFactory(io.trino.sql.analyzer.StatementAnalyzerFactory) TablePropertyManager(io.trino.metadata.TablePropertyManager) ExpressionDeserializer(io.trino.server.ExpressionSerialization.ExpressionDeserializer) MaterializedViewPropertyManager(io.trino.metadata.MaterializedViewPropertyManager) UNIFORM(io.trino.execution.scheduler.NodeSchedulerConfig.NodeSchedulerPolicy.UNIFORM) GcMonitor(io.airlift.stats.GcMonitor) OptimizerConfig(io.trino.sql.planner.OptimizerConfig) NodeMemoryConfig(io.trino.memory.NodeMemoryConfig) SystemSecurityMetadata(io.trino.metadata.SystemSecurityMetadata) NodeTaskMap(io.trino.execution.NodeTaskMap) JoinFilterFunctionCompiler(io.trino.sql.gen.JoinFilterFunctionCompiler) InternalTypeManager(io.trino.type.InternalTypeManager) SplitManager(io.trino.split.SplitManager) NodeSpillConfig(io.trino.spiller.NodeSpillConfig) JsonBinder.jsonBinder(io.airlift.json.JsonBinder.jsonBinder) Duration(io.airlift.units.Duration) PreDestroy(javax.annotation.PreDestroy) DisabledSystemSecurityMetadata(io.trino.metadata.DisabledSystemSecurityMetadata) Block(io.trino.spi.block.Block) ConditionalModule.conditionalModule(io.airlift.configuration.ConditionalModule.conditionalModule) PageSorter(io.trino.spi.PageSorter) ExportBinder.newExporter(org.weakref.jmx.guice.ExportBinder.newExporter) LiteralFunction(io.trino.metadata.LiteralFunction) SqlParser(io.trino.sql.parser.SqlParser) FailureInjector(io.trino.execution.FailureInjector) OptionalBinder.newOptionalBinder(com.google.inject.multibindings.OptionalBinder.newOptionalBinder) TaskManagementExecutor(io.trino.execution.TaskManagementExecutor) TypeSignature(io.trino.spi.type.TypeSignature) ProvidesIntoSet(com.google.inject.multibindings.ProvidesIntoSet) PageFunctionCompiler(io.trino.sql.gen.PageFunctionCompiler) SliceSerializer(io.trino.server.SliceSerialization.SliceSerializer) DirectExchangeClientSupplier(io.trino.operator.DirectExchangeClientSupplier) SplitMonitor(io.trino.event.SplitMonitor) SessionPropertyManager(io.trino.metadata.SessionPropertyManager) DataSize(io.airlift.units.DataSize) QueryManagerConfig(io.trino.execution.QueryManagerConfig) List(java.util.List) LocationFactory(io.trino.execution.LocationFactory) DirectExchangeClientFactory(io.trino.operator.DirectExchangeClientFactory) JaxrsBinder.jaxrsBinder(io.airlift.jaxrs.JaxrsBinder.jaxrsBinder) Expression(io.trino.sql.tree.Expression) ColumnPropertyManager(io.trino.metadata.ColumnPropertyManager) PageSourceProvider(io.trino.split.PageSourceProvider) TypeSignatureDeserializer(io.trino.type.TypeSignatureDeserializer) PlannerContext(io.trino.sql.PlannerContext) GlobalFunctionCatalog(io.trino.metadata.GlobalFunctionCatalog) StaticCatalogStoreConfig(io.trino.metadata.StaticCatalogStoreConfig) OperatorFactories(io.trino.operator.OperatorFactories) FileSingleStreamSpillerFactory(io.trino.spiller.FileSingleStreamSpillerFactory) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) Type(io.trino.spi.type.Type) Singleton(javax.inject.Singleton) MemoryManagerConfig(io.trino.memory.MemoryManagerConfig) ExpressionCompiler(io.trino.sql.gen.ExpressionCompiler) Inject(javax.inject.Inject) AbstractConfigurationAwareModule(io.airlift.configuration.AbstractConfigurationAwareModule) BlockEncodingManager(io.trino.metadata.BlockEncodingManager) ImmutableList(com.google.common.collect.ImmutableList) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) Objects.requireNonNull(java.util.Objects.requireNonNull) SystemSessionProperties(io.trino.SystemSessionProperties) ForExchange(io.trino.operator.ForExchange) GenericPartitioningSpillerFactory(io.trino.spiller.GenericPartitioningSpillerFactory) DynamicFilterConfig(io.trino.execution.DynamicFilterConfig) TableProceduresRegistry(io.trino.metadata.TableProceduresRegistry) IndexManager(io.trino.index.IndexManager) TypeOperatorsCache(io.trino.type.TypeOperatorsCache) ExecutorService(java.util.concurrent.ExecutorService) PagesIndex(io.trino.operator.PagesIndex) SchemaPropertyManager(io.trino.metadata.SchemaPropertyManager) DispatchManager(io.trino.dispatcher.DispatchManager) TableExecuteContextManager(io.trino.execution.TableExecuteContextManager) UniformNodeSelectorModule(io.trino.execution.scheduler.UniformNodeSelectorModule) ForNodeManager(io.trino.metadata.ForNodeManager) HandleJsonModule(io.trino.metadata.HandleJsonModule) SingleStreamSpillerFactory(io.trino.spiller.SingleStreamSpillerFactory) DiscoveryNodeManager(io.trino.metadata.DiscoveryNodeManager) TOPOLOGY(io.trino.execution.scheduler.NodeSchedulerConfig.NodeSchedulerPolicy.TOPOLOGY) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PageSourceManager(io.trino.split.PageSourceManager) Scopes(com.google.inject.Scopes) FunctionManager(io.trino.metadata.FunctionManager) CompilerConfig(io.trino.sql.planner.CompilerConfig) PartitioningSpillerFactory(io.trino.spiller.PartitioningSpillerFactory) Provides(com.google.inject.Provides) FeaturesConfig(io.trino.FeaturesConfig) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) CatalogManager(io.trino.metadata.CatalogManager) ProcedureRegistry(io.trino.metadata.ProcedureRegistry) Metadata(io.trino.metadata.Metadata) BlockEncodingSerde(io.trino.spi.block.BlockEncodingSerde) TypeManager(io.trino.spi.type.TypeManager) TypeDeserializer(io.trino.type.TypeDeserializer) AnalyzePropertyManager(io.trino.metadata.AnalyzePropertyManager) SECONDS(java.util.concurrent.TimeUnit.SECONDS) LocalMemoryManager(io.trino.memory.LocalMemoryManager) PluginsProvider(io.trino.server.PluginManager.PluginsProvider) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) FailureInjector(io.trino.execution.FailureInjector) NodeVersion(io.trino.client.NodeVersion) TypeSignature(io.trino.spi.type.TypeSignature) TableExecuteContextManager(io.trino.execution.TableExecuteContextManager) InternalTypeManager(io.trino.type.InternalTypeManager) FunctionManager(io.trino.metadata.FunctionManager) JoinCompiler(io.trino.sql.gen.JoinCompiler) GenericPartitioningSpillerFactory(io.trino.spiller.GenericPartitioningSpillerFactory) TopologyAwareNodeSelectorModule(io.trino.execution.scheduler.TopologyAwareNodeSelectorModule) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) OperatorFactories(io.trino.operator.OperatorFactories) StatementAnalyzerFactory(io.trino.sql.analyzer.StatementAnalyzerFactory) BlockJsonSerde(io.trino.block.BlockJsonSerde) MultilevelSplitQueue(io.trino.execution.executor.MultilevelSplitQueue) TypeOperatorsCache(io.trino.type.TypeOperatorsCache) JmxGcMonitor(io.airlift.stats.JmxGcMonitor) GcMonitor(io.airlift.stats.GcMonitor) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) PagesIndex(io.trino.operator.PagesIndex) BlockTypeOperators(io.trino.type.BlockTypeOperators) LocalMemoryManager(io.trino.memory.LocalMemoryManager) SystemConnectorModule(io.trino.connector.system.SystemConnectorModule) EmbedVersion(io.trino.version.EmbedVersion) PageSinkManager(io.trino.split.PageSinkManager) JoinFilterFunctionCompiler(io.trino.sql.gen.JoinFilterFunctionCompiler) VersionEmbedder(io.trino.spi.VersionEmbedder) ConnectorManager(io.trino.connector.ConnectorManager) TypeAnalyzer(io.trino.sql.planner.TypeAnalyzer) CatalogManager(io.trino.metadata.CatalogManager) UniformNodeSelectorModule(io.trino.execution.scheduler.UniformNodeSelectorModule) Type(io.trino.spi.type.Type) PlannerContext(io.trino.sql.PlannerContext) Expression(io.trino.sql.tree.Expression) FinalizerService(io.trino.util.FinalizerService) TaskManagementExecutor(io.trino.execution.TaskManagementExecutor) MemoryRevokingScheduler(io.trino.execution.MemoryRevokingScheduler) SystemSecurityMetadata(io.trino.metadata.SystemSecurityMetadata) DisabledSystemSecurityMetadata(io.trino.metadata.DisabledSystemSecurityMetadata) InternalBlockEncodingSerde(io.trino.metadata.InternalBlockEncodingSerde) TablePropertyManager(io.trino.metadata.TablePropertyManager) TypeRegistry(io.trino.metadata.TypeRegistry) PageSourceManager(io.trino.split.PageSourceManager) HttpServerConfig(io.airlift.http.server.HttpServerConfig) PagesIndexPageSorter(io.trino.operator.PagesIndexPageSorter) FileSingleStreamSpillerFactory(io.trino.spiller.FileSingleStreamSpillerFactory) SessionPropertyManager(io.trino.metadata.SessionPropertyManager) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) TypeOperators(io.trino.spi.type.TypeOperators) BlockTypeOperators(io.trino.type.BlockTypeOperators) ForExchange(io.trino.operator.ForExchange) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) StaticCatalogStore(io.trino.metadata.StaticCatalogStore) NodeTaskMap(io.trino.execution.NodeTaskMap) SplitMonitor(io.trino.event.SplitMonitor) HttpLocationFactory(io.trino.server.remotetask.HttpLocationFactory) SqlParser(io.trino.sql.parser.SqlParser) Duration(io.airlift.units.Duration) PauseMeter(io.airlift.stats.PauseMeter) SpillerStats(io.trino.spiller.SpillerStats) IndexManager(io.trino.index.IndexManager) LocalSpillManager(io.trino.spiller.LocalSpillManager) SqlTaskManager(io.trino.execution.SqlTaskManager) TaskExecutor(io.trino.execution.executor.TaskExecutor) BlockEncodingManager(io.trino.metadata.BlockEncodingManager) Slice(io.airlift.slice.Slice) Block(io.trino.spi.block.Block) ExpressionCompiler(io.trino.sql.gen.ExpressionCompiler) GroupByHashPageIndexerFactory(io.trino.operator.GroupByHashPageIndexerFactory) ColumnPropertyManager(io.trino.metadata.ColumnPropertyManager) PageFunctionCompiler(io.trino.sql.gen.PageFunctionCompiler) MaterializedViewPropertyManager(io.trino.metadata.MaterializedViewPropertyManager) SpillerFactory(io.trino.spiller.SpillerFactory) GroupByHashPageIndexerFactory(io.trino.operator.GroupByHashPageIndexerFactory) PageIndexerFactory(io.trino.spi.PageIndexerFactory) HttpLocationFactory(io.trino.server.remotetask.HttpLocationFactory) StatementAnalyzerFactory(io.trino.sql.analyzer.StatementAnalyzerFactory) LocationFactory(io.trino.execution.LocationFactory) DirectExchangeClientFactory(io.trino.operator.DirectExchangeClientFactory) FileSingleStreamSpillerFactory(io.trino.spiller.FileSingleStreamSpillerFactory) GenericPartitioningSpillerFactory(io.trino.spiller.GenericPartitioningSpillerFactory) SingleStreamSpillerFactory(io.trino.spiller.SingleStreamSpillerFactory) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PartitioningSpillerFactory(io.trino.spiller.PartitioningSpillerFactory) DisabledSystemSecurityMetadata(io.trino.metadata.DisabledSystemSecurityMetadata) LocalMemoryManagerExporter(io.trino.memory.LocalMemoryManagerExporter) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) TableProceduresRegistry(io.trino.metadata.TableProceduresRegistry) HandleJsonModule(io.trino.metadata.HandleJsonModule) SchemaPropertyManager(io.trino.metadata.SchemaPropertyManager) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) AnalyzePropertyManager(io.trino.metadata.AnalyzePropertyManager) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) ProcedureRegistry(io.trino.metadata.ProcedureRegistry) DirectExchangeClientFactory(io.trino.operator.DirectExchangeClientFactory) SpillerFactory(io.trino.spiller.SpillerFactory) FileSingleStreamSpillerFactory(io.trino.spiller.FileSingleStreamSpillerFactory) GenericPartitioningSpillerFactory(io.trino.spiller.GenericPartitioningSpillerFactory) SingleStreamSpillerFactory(io.trino.spiller.SingleStreamSpillerFactory) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PartitioningSpillerFactory(io.trino.spiller.PartitioningSpillerFactory) GlobalFunctionCatalog(io.trino.metadata.GlobalFunctionCatalog) SplitManager(io.trino.split.SplitManager) PreparedStatementEncoder(io.trino.server.protocol.PreparedStatementEncoder) MetadataManager(io.trino.metadata.MetadataManager) JmxGcMonitor(io.airlift.stats.JmxGcMonitor) DiscoveryNodeManager(io.trino.metadata.DiscoveryNodeManager) SystemSessionProperties(io.trino.SystemSessionProperties) TableProceduresPropertyManager(io.trino.metadata.TableProceduresPropertyManager)

Example 2 with NodeScheduler

use of io.trino.execution.scheduler.NodeScheduler in project trino by trinodb.

the class NodePartitioningManager method getNodePartitioningMap.

public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHandle partitioningHandle) {
    requireNonNull(session, "session is null");
    requireNonNull(partitioningHandle, "partitioningHandle is null");
    if (partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle) {
        return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getNodePartitionMap(session, nodeScheduler);
    }
    ConnectorBucketNodeMap connectorBucketNodeMap = getConnectorBucketNodeMap(session, partitioningHandle);
    // safety check for crazy partitioning
    checkArgument(connectorBucketNodeMap.getBucketCount() < 1_000_000, "Too many buckets in partitioning: %s", connectorBucketNodeMap.getBucketCount());
    List<InternalNode> bucketToNode;
    if (connectorBucketNodeMap.hasFixedMapping()) {
        bucketToNode = getFixedMapping(connectorBucketNodeMap);
    } else {
        CatalogName catalogName = partitioningHandle.getConnectorId().orElseThrow(() -> new IllegalArgumentException("No connector ID for partitioning handle: " + partitioningHandle));
        bucketToNode = createArbitraryBucketToNode(nodeScheduler.createNodeSelector(session, Optional.of(catalogName)).allNodes(), connectorBucketNodeMap.getBucketCount());
    }
    int[] bucketToPartition = new int[connectorBucketNodeMap.getBucketCount()];
    BiMap<InternalNode, Integer> nodeToPartition = HashBiMap.create();
    int nextPartitionId = 0;
    for (int bucket = 0; bucket < bucketToNode.size(); bucket++) {
        InternalNode node = bucketToNode.get(bucket);
        Integer partitionId = nodeToPartition.get(node);
        if (partitionId == null) {
            partitionId = nextPartitionId++;
            nodeToPartition.put(node, partitionId);
        }
        bucketToPartition[bucket] = partitionId;
    }
    List<InternalNode> partitionToNode = IntStream.range(0, nodeToPartition.size()).mapToObj(partitionId -> nodeToPartition.inverse().get(partitionId)).collect(toImmutableList());
    return new NodePartitionMap(partitionToNode, bucketToPartition, getSplitToBucket(session, partitioningHandle));
}
Also used : IntStream(java.util.stream.IntStream) BucketFunction(io.trino.spi.connector.BucketFunction) BucketNodeMap(io.trino.execution.scheduler.BucketNodeMap) Type(io.trino.spi.type.Type) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) Inject(javax.inject.Inject) FixedBucketNodeMap(io.trino.execution.scheduler.FixedBucketNodeMap) CatalogName(io.trino.connector.CatalogName) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ConnectorPartitionHandle(io.trino.spi.connector.ConnectorPartitionHandle) Objects.requireNonNull(java.util.Objects.requireNonNull) DynamicBucketNodeMap(io.trino.execution.scheduler.group.DynamicBucketNodeMap) BiMap(com.google.common.collect.BiMap) EmptySplit(io.trino.split.EmptySplit) PartitionFunction(io.trino.operator.PartitionFunction) ConnectorNodePartitioningProvider(io.trino.spi.connector.ConnectorNodePartitioningProvider) BlockTypeOperators(io.trino.type.BlockTypeOperators) ConnectorSplit(io.trino.spi.connector.ConnectorSplit) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ToIntFunction(java.util.function.ToIntFunction) BucketPartitionFunction(io.trino.operator.BucketPartitionFunction) ConnectorBucketNodeMap(io.trino.spi.connector.ConnectorBucketNodeMap) HashBiMap(com.google.common.collect.HashBiMap) InternalNode(io.trino.metadata.InternalNode) List(java.util.List) Stream(java.util.stream.Stream) Split(io.trino.metadata.Split) Optional(java.util.Optional) Collections(java.util.Collections) Session(io.trino.Session) ConnectorBucketNodeMap(io.trino.spi.connector.ConnectorBucketNodeMap) CatalogName(io.trino.connector.CatalogName) InternalNode(io.trino.metadata.InternalNode)

Example 3 with NodeScheduler

use of io.trino.execution.scheduler.NodeScheduler in project trino by trinodb.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    PageSourceManager pageSourceManager = new PageSourceManager();
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(PLANNER_CONTEXT.getTypeOperators());
    NodeScheduler nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, blockTypeOperators);
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(PLANNER_CONTEXT.getFunctionManager(), 0);
    return new LocalExecutionPlanner(PLANNER_CONTEXT, createTestingTypeAnalyzer(PLANNER_CONTEXT), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockDirectExchangeClientSupplier(), new ExpressionCompiler(PLANNER_CONTEXT.getFunctionManager(), pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(PLANNER_CONTEXT.getFunctionManager()), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new PagesIndex.TestingFactory(false), new JoinCompiler(PLANNER_CONTEXT.getTypeOperators()), new TrinoOperatorFactories(), new OrderingCompiler(PLANNER_CONTEXT.getTypeOperators()), new DynamicFilterConfig(), blockTypeOperators, new TableExecuteContextManager(), new ExchangeManagerRegistry(new ExchangeHandleResolver()));
}
Also used : PlanFragment(io.trino.sql.planner.PlanFragment) JoinFilterFunctionCompiler(io.trino.sql.gen.JoinFilterFunctionCompiler) ObjectMapperProvider(io.airlift.json.ObjectMapperProvider) CatalogName(io.trino.connector.CatalogName) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) FinalizerService(io.trino.util.FinalizerService) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) TEST_SESSION(io.trino.SessionTestUtils.TEST_SESSION) SINGLE_DISTRIBUTION(io.trino.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) TableScanNode(io.trino.sql.planner.plan.TableScanNode) ImmutableMap(com.google.common.collect.ImmutableMap) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) PageSinkManager(io.trino.split.PageSinkManager) PageFunctionCompiler(io.trino.sql.gen.PageFunctionCompiler) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) UniformNodeSelectorFactory(io.trino.execution.scheduler.UniformNodeSelectorFactory) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) SplitMonitor(io.trino.event.SplitMonitor) MockDirectExchangeClientSupplier(io.trino.execution.TestSqlTaskManager.MockDirectExchangeClientSupplier) List(java.util.List) BIGINT(io.trino.spi.type.BigintType.BIGINT) Split(io.trino.metadata.Split) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) Optional(java.util.Optional) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) TestingColumnHandle(io.trino.testing.TestingMetadata.TestingColumnHandle) TypeAnalyzer.createTestingTypeAnalyzer(io.trino.sql.planner.TypeAnalyzer.createTestingTypeAnalyzer) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) JoinCompiler(io.trino.sql.gen.JoinCompiler) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) Partitioning(io.trino.sql.planner.Partitioning) PartitioningScheme(io.trino.sql.planner.PartitioningScheme) StatsAndCosts(io.trino.cost.StatsAndCosts) ExpressionCompiler(io.trino.sql.gen.ExpressionCompiler) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) TestingSplit(io.trino.testing.TestingSplit) StageExecutionDescriptor.ungroupedExecution(io.trino.operator.StageExecutionDescriptor.ungroupedExecution) EventListenerConfig(io.trino.eventlistener.EventListenerConfig) IndexManager(io.trino.index.IndexManager) PagesIndex(io.trino.operator.PagesIndex) Symbol(io.trino.sql.planner.Symbol) BlockTypeOperators(io.trino.type.BlockTypeOperators) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PageSourceManager(io.trino.split.PageSourceManager) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) PLANNER_CONTEXT(io.trino.sql.planner.TestingPlannerContext.PLANNER_CONTEXT) EventListenerManager(io.trino.eventlistener.EventListenerManager) TEST_TABLE_HANDLE(io.trino.testing.TestingHandles.TEST_TABLE_HANDLE) OutputBuffers(io.trino.execution.buffer.OutputBuffers) SOURCE_DISTRIBUTION(io.trino.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) PageFunctionCompiler(io.trino.sql.gen.PageFunctionCompiler) MockDirectExchangeClientSupplier(io.trino.execution.TestSqlTaskManager.MockDirectExchangeClientSupplier) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) PagesIndex(io.trino.operator.PagesIndex) PageSourceManager(io.trino.split.PageSourceManager) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) BlockTypeOperators(io.trino.type.BlockTypeOperators) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PageSinkManager(io.trino.split.PageSinkManager) JoinCompiler(io.trino.sql.gen.JoinCompiler) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) UniformNodeSelectorFactory(io.trino.execution.scheduler.UniformNodeSelectorFactory) JoinFilterFunctionCompiler(io.trino.sql.gen.JoinFilterFunctionCompiler) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) IndexManager(io.trino.index.IndexManager) FinalizerService(io.trino.util.FinalizerService) ExpressionCompiler(io.trino.sql.gen.ExpressionCompiler)

Example 4 with NodeScheduler

use of io.trino.execution.scheduler.NodeScheduler in project trino by trinodb.

the class TestLocalExchange method setUp.

@BeforeMethod
public void setUp() {
    NodeScheduler nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(new FinalizerService())));
    nodePartitioningManager = new NodePartitioningManager(nodeScheduler, new BlockTypeOperators(new TypeOperators()));
}
Also used : BlockTypeOperators(io.trino.type.BlockTypeOperators) NodeTaskMap(io.trino.execution.NodeTaskMap) FinalizerService(io.trino.util.FinalizerService) UniformNodeSelectorFactory(io.trino.execution.scheduler.UniformNodeSelectorFactory) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) TypeOperators(io.trino.spi.type.TypeOperators) BlockTypeOperators(io.trino.type.BlockTypeOperators) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 5 with NodeScheduler

use of io.trino.execution.scheduler.NodeScheduler in project trino by trinodb.

the class TestNodeScheduler method testTopologyAwareScheduling.

@Test(timeOut = 60 * 1000)
public void testTopologyAwareScheduling() {
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    ImmutableList.Builder<InternalNode> nodeBuilder = ImmutableList.builder();
    nodeBuilder.add(new InternalNode("node1", URI.create("http://host1.rack1:11"), NodeVersion.UNKNOWN, false));
    nodeBuilder.add(new InternalNode("node2", URI.create("http://host2.rack1:12"), NodeVersion.UNKNOWN, false));
    nodeBuilder.add(new InternalNode("node3", URI.create("http://host3.rack2:13"), NodeVersion.UNKNOWN, false));
    ImmutableList<InternalNode> nodes = nodeBuilder.build();
    nodeManager.addNode(CONNECTOR_ID, nodes);
    // contents of taskMap indicate the node-task map for the current stage
    Map<InternalNode, RemoteTask> taskMap = new HashMap<>();
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(25).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(20);
    TestNetworkTopology topology = new TestNetworkTopology();
    NodeSelectorFactory nodeSelectorFactory = new TopologyAwareNodeSelectorFactory(topology, nodeManager, nodeSchedulerConfig, nodeTaskMap, getNetworkTopologyConfig());
    NodeScheduler nodeScheduler = new NodeScheduler(nodeSelectorFactory);
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, Optional.of(CONNECTOR_ID));
    // Fill up the nodes with non-local data
    ImmutableSet.Builder<Split> nonRackLocalBuilder = ImmutableSet.builder();
    for (int i = 0; i < (25 + 11) * 3; i++) {
        nonRackLocalBuilder.add(new Split(CONNECTOR_ID, new TestSplitRemote(HostAddress.fromParts("data.other_rack", 1)), Lifespan.taskWide()));
    }
    Set<Split> nonRackLocalSplits = nonRackLocalBuilder.build();
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(nonRackLocalSplits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    int task = 0;
    for (InternalNode node : assignments.keySet()) {
        TaskId taskId = new TaskId(new StageId("test", 1), task, 0);
        task++;
        MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, ImmutableList.copyOf(assignments.get(node)), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId));
        remoteTask.startSplits(25);
        nodeTaskMap.addTask(node, remoteTask);
        taskMap.put(node, remoteTask);
    }
    // Continue assigning to fill up part of the queue
    nonRackLocalSplits = Sets.difference(nonRackLocalSplits, new HashSet<>(assignments.values()));
    assignments = nodeSelector.computeAssignments(nonRackLocalSplits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    for (InternalNode node : assignments.keySet()) {
        RemoteTask remoteTask = taskMap.get(node);
        remoteTask.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), assignments.get(node)).build());
    }
    nonRackLocalSplits = Sets.difference(nonRackLocalSplits, new HashSet<>(assignments.values()));
    // Check that 3 of the splits were rejected, since they're non-local
    assertEquals(nonRackLocalSplits.size(), 3);
    // Assign rack-local splits
    ImmutableSet.Builder<Split> rackLocalSplits = ImmutableSet.builder();
    HostAddress dataHost1 = HostAddress.fromParts("data.rack1", 1);
    HostAddress dataHost2 = HostAddress.fromParts("data.rack2", 1);
    for (int i = 0; i < 6 * 2; i++) {
        rackLocalSplits.add(new Split(CONNECTOR_ID, new TestSplitRemote(dataHost1), Lifespan.taskWide()));
    }
    for (int i = 0; i < 6; i++) {
        rackLocalSplits.add(new Split(CONNECTOR_ID, new TestSplitRemote(dataHost2), Lifespan.taskWide()));
    }
    assignments = nodeSelector.computeAssignments(rackLocalSplits.build(), ImmutableList.copyOf(taskMap.values())).getAssignments();
    for (InternalNode node : assignments.keySet()) {
        RemoteTask remoteTask = taskMap.get(node);
        remoteTask.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), assignments.get(node)).build());
    }
    Set<Split> unassigned = Sets.difference(rackLocalSplits.build(), new HashSet<>(assignments.values()));
    // Compute the assignments a second time to account for the fact that some splits may not have been assigned due to asynchronous
    // loading of the NetworkLocationCache
    assignments = nodeSelector.computeAssignments(unassigned, ImmutableList.copyOf(taskMap.values())).getAssignments();
    for (InternalNode node : assignments.keySet()) {
        RemoteTask remoteTask = taskMap.get(node);
        remoteTask.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), assignments.get(node)).build());
    }
    unassigned = Sets.difference(unassigned, new HashSet<>(assignments.values()));
    assertEquals(unassigned.size(), 3);
    int rack1 = 0;
    int rack2 = 0;
    for (Split split : unassigned) {
        String rack = topology.locate(split.getAddresses().get(0)).getSegments().get(0);
        switch(rack) {
            case "rack1":
                rack1++;
                break;
            case "rack2":
                rack2++;
                break;
            default:
                throw new AssertionError("Unexpected rack: " + rack);
        }
    }
    assertEquals(rack1, 2);
    assertEquals(rack2, 1);
    // Assign local splits
    ImmutableSet.Builder<Split> localSplits = ImmutableSet.builder();
    localSplits.add(new Split(CONNECTOR_ID, new TestSplitRemote(HostAddress.fromParts("host1.rack1", 1)), Lifespan.taskWide()));
    localSplits.add(new Split(CONNECTOR_ID, new TestSplitRemote(HostAddress.fromParts("host2.rack1", 1)), Lifespan.taskWide()));
    localSplits.add(new Split(CONNECTOR_ID, new TestSplitRemote(HostAddress.fromParts("host3.rack2", 1)), Lifespan.taskWide()));
    assignments = nodeSelector.computeAssignments(localSplits.build(), ImmutableList.copyOf(taskMap.values())).getAssignments();
    assertEquals(assignments.size(), 3);
    assertEquals(assignments.keySet().size(), 3);
}
Also used : HashMap(java.util.HashMap) ImmutableList(com.google.common.collect.ImmutableList) TopologyAwareNodeSelectorFactory(io.trino.execution.scheduler.TopologyAwareNodeSelectorFactory) NodeSelectorFactory(io.trino.execution.scheduler.NodeSelectorFactory) UniformNodeSelectorFactory(io.trino.execution.scheduler.UniformNodeSelectorFactory) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) HostAddress(io.trino.spi.HostAddress) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) ImmutableSet(com.google.common.collect.ImmutableSet) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) InternalNode(io.trino.metadata.InternalNode) NodeSelector(io.trino.execution.scheduler.NodeSelector) UniformNodeSelector(io.trino.execution.scheduler.UniformNodeSelector) Split(io.trino.metadata.Split) ConnectorSplit(io.trino.spi.connector.ConnectorSplit) TopologyAwareNodeSelectorFactory(io.trino.execution.scheduler.TopologyAwareNodeSelectorFactory) Test(org.testng.annotations.Test)

Aggregations

NodeScheduler (io.trino.execution.scheduler.NodeScheduler)7 NodeSchedulerConfig (io.trino.execution.scheduler.NodeSchedulerConfig)6 UniformNodeSelectorFactory (io.trino.execution.scheduler.UniformNodeSelectorFactory)5 InMemoryNodeManager (io.trino.metadata.InMemoryNodeManager)5 BlockTypeOperators (io.trino.type.BlockTypeOperators)5 FinalizerService (io.trino.util.FinalizerService)5 NodePartitioningManager (io.trino.sql.planner.NodePartitioningManager)4 ImmutableList (com.google.common.collect.ImmutableList)3 NodeTaskMap (io.trino.execution.NodeTaskMap)3 Split (io.trino.metadata.Split)3 TypeOperators (io.trino.spi.type.TypeOperators)3 List (java.util.List)3 CatalogName (io.trino.connector.CatalogName)2 SplitMonitor (io.trino.event.SplitMonitor)2 IndexManager (io.trino.index.IndexManager)2 PagesIndex (io.trino.operator.PagesIndex)2 TrinoOperatorFactories (io.trino.operator.TrinoOperatorFactories)2 IndexJoinLookupStats (io.trino.operator.index.IndexJoinLookupStats)2 Type (io.trino.spi.type.Type)2 GenericSpillerFactory (io.trino.spiller.GenericSpillerFactory)2