use of io.trino.sql.planner.NodePartitioningManager in project trino by trinodb.
the class ServerMainModule method setup.
@Override
protected void setup(Binder binder) {
ServerConfig serverConfig = buildConfigObject(ServerConfig.class);
if (serverConfig.isCoordinator()) {
install(new CoordinatorModule());
} else {
install(new WorkerModule());
}
binder.bind(StartupStatus.class).in(Scopes.SINGLETON);
configBinder(binder).bindConfigDefaults(HttpServerConfig.class, httpServerConfig -> {
httpServerConfig.setAdminEnabled(false);
});
binder.bind(PreparedStatementEncoder.class).in(Scopes.SINGLETON);
binder.bind(HttpRequestSessionContextFactory.class).in(Scopes.SINGLETON);
install(new InternalCommunicationModule());
configBinder(binder).bindConfig(FeaturesConfig.class);
configBinder(binder).bindConfig(OptimizerConfig.class);
configBinder(binder).bindConfig(ProtocolConfig.class);
binder.bind(SqlParser.class).in(Scopes.SINGLETON);
jaxrsBinder(binder).bind(ThrowableMapper.class);
configBinder(binder).bindConfig(QueryManagerConfig.class);
configBinder(binder).bindConfig(SqlEnvironmentConfig.class);
newOptionalBinder(binder, ExplainAnalyzeContext.class);
binder.bind(StatementAnalyzerFactory.class).in(Scopes.SINGLETON);
// GC Monitor
binder.bind(GcMonitor.class).to(JmxGcMonitor.class).in(Scopes.SINGLETON);
newExporter(binder).export(GcMonitor.class).withGeneratedName();
// session properties
newSetBinder(binder, SystemSessionPropertiesProvider.class).addBinding().to(SystemSessionProperties.class);
binder.bind(SessionPropertyManager.class).in(Scopes.SINGLETON);
binder.bind(SystemSessionProperties.class).in(Scopes.SINGLETON);
binder.bind(SessionPropertyDefaults.class).in(Scopes.SINGLETON);
// schema properties
binder.bind(SchemaPropertyManager.class).in(Scopes.SINGLETON);
// table properties
binder.bind(TablePropertyManager.class).in(Scopes.SINGLETON);
// materialized view properties
binder.bind(MaterializedViewPropertyManager.class).in(Scopes.SINGLETON);
// column properties
binder.bind(ColumnPropertyManager.class).in(Scopes.SINGLETON);
// analyze properties
binder.bind(AnalyzePropertyManager.class).in(Scopes.SINGLETON);
// table procedures properties
binder.bind(TableProceduresPropertyManager.class).in(Scopes.SINGLETON);
// node manager
discoveryBinder(binder).bindSelector("trino");
binder.bind(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
binder.bind(InternalNodeManager.class).to(DiscoveryNodeManager.class).in(Scopes.SINGLETON);
newExporter(binder).export(DiscoveryNodeManager.class).withGeneratedName();
httpClientBinder(binder).bindHttpClient("node-manager", ForNodeManager.class).withTracing().withConfigDefaults(config -> {
config.setIdleTimeout(new Duration(30, SECONDS));
config.setRequestTimeout(new Duration(10, SECONDS));
});
// node scheduler
// TODO: remove from NodePartitioningManager and move to CoordinatorModule
configBinder(binder).bindConfig(NodeSchedulerConfig.class);
binder.bind(NodeScheduler.class).in(Scopes.SINGLETON);
binder.bind(NodeTaskMap.class).in(Scopes.SINGLETON);
newExporter(binder).export(NodeScheduler.class).withGeneratedName();
// network topology
// TODO: move to CoordinatorModule when NodeScheduler is moved
install(conditionalModule(NodeSchedulerConfig.class, config -> UNIFORM == config.getNodeSchedulerPolicy(), new UniformNodeSelectorModule()));
install(conditionalModule(NodeSchedulerConfig.class, config -> TOPOLOGY == config.getNodeSchedulerPolicy(), new TopologyAwareNodeSelectorModule()));
// task execution
configBinder(binder).bindConfig(FailureInjectionConfig.class);
binder.bind(FailureInjector.class).in(Scopes.SINGLETON);
jaxrsBinder(binder).bind(TaskResource.class);
newExporter(binder).export(TaskResource.class).withGeneratedName();
jaxrsBinder(binder).bind(TaskExecutorResource.class);
newExporter(binder).export(TaskExecutorResource.class).withGeneratedName();
binder.bind(TaskManagementExecutor.class).in(Scopes.SINGLETON);
binder.bind(SqlTaskManager.class).in(Scopes.SINGLETON);
binder.bind(TableExecuteContextManager.class).in(Scopes.SINGLETON);
// memory revoking scheduler
binder.bind(MemoryRevokingScheduler.class).in(Scopes.SINGLETON);
// Add monitoring for JVM pauses
binder.bind(PauseMeter.class).in(Scopes.SINGLETON);
newExporter(binder).export(PauseMeter.class).withGeneratedName();
configBinder(binder).bindConfig(MemoryManagerConfig.class);
configBinder(binder).bindConfig(NodeMemoryConfig.class);
binder.bind(LocalMemoryManager.class).in(Scopes.SINGLETON);
binder.bind(LocalMemoryManagerExporter.class).in(Scopes.SINGLETON);
newOptionalBinder(binder, VersionEmbedder.class).setDefault().to(EmbedVersion.class).in(Scopes.SINGLETON);
newExporter(binder).export(SqlTaskManager.class).withGeneratedName();
binder.bind(TaskExecutor.class).in(Scopes.SINGLETON);
newExporter(binder).export(TaskExecutor.class).withGeneratedName();
binder.bind(MultilevelSplitQueue.class).in(Scopes.SINGLETON);
newExporter(binder).export(MultilevelSplitQueue.class).withGeneratedName();
binder.bind(LocalExecutionPlanner.class).in(Scopes.SINGLETON);
configBinder(binder).bindConfig(CompilerConfig.class);
binder.bind(ExpressionCompiler.class).in(Scopes.SINGLETON);
newExporter(binder).export(ExpressionCompiler.class).withGeneratedName();
binder.bind(PageFunctionCompiler.class).in(Scopes.SINGLETON);
newExporter(binder).export(PageFunctionCompiler.class).withGeneratedName();
configBinder(binder).bindConfig(TaskManagerConfig.class);
binder.bind(IndexJoinLookupStats.class).in(Scopes.SINGLETON);
newExporter(binder).export(IndexJoinLookupStats.class).withGeneratedName();
binder.bind(AsyncHttpExecutionMBean.class).in(Scopes.SINGLETON);
newExporter(binder).export(AsyncHttpExecutionMBean.class).withGeneratedName();
binder.bind(JoinFilterFunctionCompiler.class).in(Scopes.SINGLETON);
newExporter(binder).export(JoinFilterFunctionCompiler.class).withGeneratedName();
binder.bind(JoinCompiler.class).in(Scopes.SINGLETON);
newExporter(binder).export(JoinCompiler.class).withGeneratedName();
binder.bind(OrderingCompiler.class).in(Scopes.SINGLETON);
newExporter(binder).export(OrderingCompiler.class).withGeneratedName();
binder.bind(PagesIndex.Factory.class).to(PagesIndex.DefaultFactory.class);
newOptionalBinder(binder, OperatorFactories.class).setDefault().to(TrinoOperatorFactories.class).in(Scopes.SINGLETON);
jaxrsBinder(binder).bind(PagesResponseWriter.class);
// exchange client
binder.bind(DirectExchangeClientSupplier.class).to(DirectExchangeClientFactory.class).in(Scopes.SINGLETON);
httpClientBinder(binder).bindHttpClient("exchange", ForExchange.class).withTracing().withFilter(GenerateTraceTokenRequestFilter.class).withConfigDefaults(config -> {
config.setIdleTimeout(new Duration(30, SECONDS));
config.setRequestTimeout(new Duration(10, SECONDS));
config.setMaxConnectionsPerServer(250);
config.setMaxContentLength(DataSize.of(32, MEGABYTE));
});
configBinder(binder).bindConfig(DirectExchangeClientConfig.class);
binder.bind(ExchangeExecutionMBean.class).in(Scopes.SINGLETON);
newExporter(binder).export(ExchangeExecutionMBean.class).withGeneratedName();
// execution
binder.bind(LocationFactory.class).to(HttpLocationFactory.class).in(Scopes.SINGLETON);
// memory manager
jaxrsBinder(binder).bind(MemoryResource.class);
jsonCodecBinder(binder).bindJsonCodec(MemoryInfo.class);
// transaction manager
configBinder(binder).bindConfig(TransactionManagerConfig.class);
// data stream provider
binder.bind(PageSourceManager.class).in(Scopes.SINGLETON);
binder.bind(PageSourceProvider.class).to(PageSourceManager.class).in(Scopes.SINGLETON);
// page sink provider
binder.bind(PageSinkManager.class).in(Scopes.SINGLETON);
binder.bind(PageSinkProvider.class).to(PageSinkManager.class).in(Scopes.SINGLETON);
// metadata
binder.bind(StaticCatalogStore.class).in(Scopes.SINGLETON);
configBinder(binder).bindConfig(StaticCatalogStoreConfig.class);
binder.bind(MetadataManager.class).in(Scopes.SINGLETON);
binder.bind(Metadata.class).to(MetadataManager.class).in(Scopes.SINGLETON);
newOptionalBinder(binder, SystemSecurityMetadata.class).setDefault().to(DisabledSystemSecurityMetadata.class).in(Scopes.SINGLETON);
binder.bind(GlobalFunctionCatalog.class).in(Scopes.SINGLETON);
binder.bind(TypeOperatorsCache.class).in(Scopes.SINGLETON);
newExporter(binder).export(TypeOperatorsCache.class).as(factory -> factory.generatedNameOf(TypeOperators.class));
binder.bind(BlockTypeOperators.class).in(Scopes.SINGLETON);
newExporter(binder).export(TypeOperatorsCache.class).withGeneratedName();
binder.bind(ProcedureRegistry.class).in(Scopes.SINGLETON);
binder.bind(TableProceduresRegistry.class).in(Scopes.SINGLETON);
binder.bind(PlannerContext.class).in(Scopes.SINGLETON);
// function
binder.bind(FunctionManager.class).in(Scopes.SINGLETON);
newSetBinder(binder, FunctionBundle.class);
binder.bind(RegisterFunctionBundles.class).asEagerSingleton();
// type
binder.bind(TypeAnalyzer.class).in(Scopes.SINGLETON);
jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
jsonBinder(binder).addDeserializerBinding(TypeSignature.class).to(TypeSignatureDeserializer.class);
jsonBinder(binder).addKeyDeserializerBinding(TypeSignature.class).to(TypeSignatureKeyDeserializer.class);
binder.bind(TypeRegistry.class).in(Scopes.SINGLETON);
binder.bind(TypeManager.class).to(InternalTypeManager.class).in(Scopes.SINGLETON);
newSetBinder(binder, Type.class);
// split manager
binder.bind(SplitManager.class).in(Scopes.SINGLETON);
// node partitioning manager
binder.bind(NodePartitioningManager.class).in(Scopes.SINGLETON);
// index manager
binder.bind(IndexManager.class).in(Scopes.SINGLETON);
// handle resolver
binder.install(new HandleJsonModule());
// connector
binder.bind(ConnectorManager.class).in(Scopes.SINGLETON);
// system connector
binder.install(new SystemConnectorModule());
// slice
jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class);
jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class);
// expression
jsonBinder(binder).addSerializerBinding(Expression.class).to(ExpressionSerializer.class);
jsonBinder(binder).addDeserializerBinding(Expression.class).to(ExpressionDeserializer.class);
// split monitor
binder.bind(SplitMonitor.class).in(Scopes.SINGLETON);
// version and announcement
binder.bind(NodeVersion.class).toInstance(new NodeVersion(nodeVersion));
discoveryBinder(binder).bindHttpAnnouncement("trino").addProperty("node_version", nodeVersion).addProperty("coordinator", String.valueOf(serverConfig.isCoordinator()));
// server info resource
jaxrsBinder(binder).bind(ServerInfoResource.class);
// node status resource
jaxrsBinder(binder).bind(StatusResource.class);
// plugin manager
binder.bind(PluginManager.class).in(Scopes.SINGLETON);
newOptionalBinder(binder, PluginsProvider.class).setDefault().to(ServerPluginsProvider.class).in(Scopes.SINGLETON);
configBinder(binder).bindConfig(ServerPluginsProviderConfig.class);
binder.bind(CatalogManager.class).in(Scopes.SINGLETON);
// block encodings
binder.bind(BlockEncodingManager.class).in(Scopes.SINGLETON);
jsonBinder(binder).addSerializerBinding(Block.class).to(BlockJsonSerde.Serializer.class);
jsonBinder(binder).addDeserializerBinding(Block.class).to(BlockJsonSerde.Deserializer.class);
binder.bind(BlockEncodingSerde.class).to(InternalBlockEncodingSerde.class).in(Scopes.SINGLETON);
// thread visualizer
jaxrsBinder(binder).bind(ThreadResource.class);
// PageSorter
binder.bind(PageSorter.class).to(PagesIndexPageSorter.class).in(Scopes.SINGLETON);
// PageIndexer
binder.bind(PageIndexerFactory.class).to(GroupByHashPageIndexerFactory.class).in(Scopes.SINGLETON);
// Finalizer
binder.bind(FinalizerService.class).in(Scopes.SINGLETON);
// Spiller
binder.bind(SpillerFactory.class).to(GenericSpillerFactory.class).in(Scopes.SINGLETON);
binder.bind(SingleStreamSpillerFactory.class).to(FileSingleStreamSpillerFactory.class).in(Scopes.SINGLETON);
binder.bind(PartitioningSpillerFactory.class).to(GenericPartitioningSpillerFactory.class).in(Scopes.SINGLETON);
binder.bind(SpillerStats.class).in(Scopes.SINGLETON);
newExporter(binder).export(SpillerFactory.class).withGeneratedName();
binder.bind(LocalSpillManager.class).in(Scopes.SINGLETON);
configBinder(binder).bindConfig(NodeSpillConfig.class);
// Dynamic Filtering
configBinder(binder).bindConfig(DynamicFilterConfig.class);
// dispatcher
// TODO remove dispatcher fromm ServerMainModule, and bind dependent components only on coordinators
newOptionalBinder(binder, DispatchManager.class);
// Added for RuleStatsSystemTable
// TODO: remove this when system tables are bound separately for coordinator and worker
newOptionalBinder(binder, RuleStatsRecorder.class);
// cleanup
binder.bind(ExecutorCleanup.class).in(Scopes.SINGLETON);
}
use of io.trino.sql.planner.NodePartitioningManager in project trino by trinodb.
the class TaskTestUtils method createTestingPlanner.
public static LocalExecutionPlanner createTestingPlanner() {
PageSourceManager pageSourceManager = new PageSourceManager();
pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
// we don't start the finalizer so nothing will be collected, which is ok for a test
FinalizerService finalizerService = new FinalizerService();
BlockTypeOperators blockTypeOperators = new BlockTypeOperators(PLANNER_CONTEXT.getTypeOperators());
NodeScheduler nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)));
NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, blockTypeOperators);
PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(PLANNER_CONTEXT.getFunctionManager(), 0);
return new LocalExecutionPlanner(PLANNER_CONTEXT, createTestingTypeAnalyzer(PLANNER_CONTEXT), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockDirectExchangeClientSupplier(), new ExpressionCompiler(PLANNER_CONTEXT.getFunctionManager(), pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(PLANNER_CONTEXT.getFunctionManager()), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}), (types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, (types, partitionFunction, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, new PagesIndex.TestingFactory(false), new JoinCompiler(PLANNER_CONTEXT.getTypeOperators()), new TrinoOperatorFactories(), new OrderingCompiler(PLANNER_CONTEXT.getTypeOperators()), new DynamicFilterConfig(), blockTypeOperators, new TableExecuteContextManager(), new ExchangeManagerRegistry(new ExchangeHandleResolver()));
}
use of io.trino.sql.planner.NodePartitioningManager in project trino by trinodb.
the class TestHashJoinOperator method testOuterJoinWithNullProbeAndFilterFunction.
@Test(dataProvider = "hashJoinTestValues")
public void testOuterJoinWithNullProbeAndFilterFunction(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
TaskContext taskContext = createTaskContext();
InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> VARCHAR.getSlice(rightPage.getBlock(0), rightPosition).toStringAscii().equals("a")));
// build factory
List<Type> buildTypes = ImmutableList.of(VARCHAR);
RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), buildTypes).row("a").row("b").row("c");
BuildSideSetup buildSideSetup = setupBuildSide(nodePartitioningManager, parallelBuild, taskContext, buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
// probe factory
List<Type> probeTypes = ImmutableList.of(VARCHAR);
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = probePages.row("a").row((String) null).row((String) null).row("a").row("b").build();
OperatorFactory joinOperatorFactory = probeOuterJoinOperatorFactory(lookupSourceFactory, probePages, true);
// build drivers and operators
instantiateBuildDrivers(buildSideSetup, taskContext);
buildLookupSource(executor, buildSideSetup);
// expected
MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("a", "a").row(null, null).row(null, null).row("a", "a").row("b", null).build();
assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}
use of io.trino.sql.planner.NodePartitioningManager in project trino by trinodb.
the class TestHashJoinOperator method testOuterJoinWithNullBuildAndFilterFunction.
@Test(dataProvider = "hashJoinTestValues")
public void testOuterJoinWithNullBuildAndFilterFunction(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
TaskContext taskContext = createTaskContext();
InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> ImmutableSet.of("a", "c").contains(VARCHAR.getSlice(rightPage.getBlock(0), rightPosition).toStringAscii())));
// build factory
List<Type> buildTypes = ImmutableList.of(VARCHAR);
RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR)).row("a").row((String) null).row((String) null).row("a").row("b");
BuildSideSetup buildSideSetup = setupBuildSide(nodePartitioningManager, parallelBuild, taskContext, buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
// probe factory
List<Type> probeTypes = ImmutableList.of(VARCHAR);
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = probePages.row("a").row("b").row("c").build();
OperatorFactory joinOperatorFactory = probeOuterJoinOperatorFactory(lookupSourceFactory, probePages, true);
// build drivers and operators
instantiateBuildDrivers(buildSideSetup, taskContext);
buildLookupSource(executor, buildSideSetup);
// expected
MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("a", "a").row("a", "a").row("b", null).row("c", null).build();
assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}
use of io.trino.sql.planner.NodePartitioningManager in project trino by trinodb.
the class TestHashJoinOperator method testProbeOuterJoinWithFilterFunction.
@Test(dataProvider = "hashJoinTestValues")
public void testProbeOuterJoinWithFilterFunction(boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) {
TaskContext taskContext = createTaskContext();
InternalJoinFilterFunction filterFunction = new TestInternalJoinFilterFunction(((leftPosition, leftPage, rightPosition, rightPage) -> BIGINT.getLong(rightPage.getBlock(1), rightPosition) >= 1025));
// build factory
List<Type> buildTypes = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)).addSequencePage(10, 20, 30, 40);
BuildSideSetup buildSideSetup = setupBuildSide(nodePartitioningManager, parallelBuild, taskContext, buildPages, Optional.of(filterFunction), false, SINGLE_STREAM_SPILLER_FACTORY);
JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactory = buildSideSetup.getLookupSourceFactoryManager();
// probe factory
List<Type> probeTypes = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes);
List<Page> probeInput = probePages.addSequencePage(15, 20, 1020, 2020).build();
OperatorFactory joinOperatorFactory = probeOuterJoinOperatorFactory(lookupSourceFactory, probePages, true);
// build drivers and operators
instantiateBuildDrivers(buildSideSetup, taskContext);
buildLookupSource(executor, buildSideSetup);
// expected
MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)).row("20", 1020L, 2020L, null, null, null).row("21", 1021L, 2021L, null, null, null).row("22", 1022L, 2022L, null, null, null).row("23", 1023L, 2023L, null, null, null).row("24", 1024L, 2024L, null, null, null).row("25", 1025L, 2025L, "25", 35L, 45L).row("26", 1026L, 2026L, "26", 36L, 46L).row("27", 1027L, 2027L, "27", 37L, 47L).row("28", 1028L, 2028L, "28", 38L, 48L).row("29", 1029L, 2029L, "29", 39L, 49L).row("30", 1030L, 2030L, null, null, null).row("31", 1031L, 2031L, null, null, null).row("32", 1032L, 2032L, null, null, null).row("33", 1033L, 2033L, null, null, null).row("34", 1034L, 2034L, null, null, null).build();
assertOperatorEquals(joinOperatorFactory, taskContext.addPipelineContext(0, true, true, false).addDriverContext(), probeInput, expected, true, getHashChannels(probePages, buildPages));
}
Aggregations