Search in sources :

Example 1 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class TestDistinctLimitOperator method setUp.

@BeforeMethod
public void setUp() {
    executor = newCachedThreadPool(daemonThreadsNamed(getClass().getSimpleName() + "-%s"));
    scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed(getClass().getSimpleName() + "-scheduledExecutor-%s"));
    driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION).addPipelineContext(0, true, true, false).addDriverContext();
    TypeOperators typeOperators = new TypeOperators();
    blockTypeOperators = new BlockTypeOperators(typeOperators);
    joinCompiler = new JoinCompiler(typeOperators);
}
Also used : JoinCompiler(io.trino.sql.gen.JoinCompiler) BlockTypeOperators(io.trino.type.BlockTypeOperators) TypeOperators(io.trino.spi.type.TypeOperators) BlockTypeOperators(io.trino.type.BlockTypeOperators) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 2 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class TestGroupedTopNRankBuilder method testYield.

@Test
public void testYield() {
    TypeOperators typeOperators = new TypeOperators();
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(typeOperators);
    List<Type> types = ImmutableList.of(BIGINT, DOUBLE);
    Page input = rowPagesBuilder(types).row(1L, 0.3).row(1L, 0.2).row(1L, 0.9).row(1L, 0.1).build().get(0);
    input.compact();
    AtomicBoolean unblock = new AtomicBoolean();
    GroupByHash groupByHash = createGroupByHash(ImmutableList.of(types.get(0)), ImmutableList.of(0), unblock::get, typeOperators, blockTypeOperators);
    GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNRankBuilder(types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST), typeOperators), new SimplePageWithPositionEqualsAndHash(types, ImmutableList.of(1), blockTypeOperators), 5, false, groupByHash);
    Work<?> work = groupedTopNBuilder.processPage(input);
    assertFalse(work.process());
    assertFalse(work.process());
    unblock.set(true);
    assertTrue(work.process());
    List<Page> output = ImmutableList.copyOf(groupedTopNBuilder.buildResult());
    assertEquals(output.size(), 1);
    Page expected = rowPagesBuilder(types).row(1L, 0.1).row(1L, 0.2).row(1L, 0.3).row(1L, 0.9).build().get(0);
    assertPageEquals(types, output.get(0), expected);
}
Also used : Page(io.trino.spi.Page) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Type(io.trino.spi.type.Type) BlockTypeOperators(io.trino.type.BlockTypeOperators) TypeOperators(io.trino.spi.type.TypeOperators) BlockTypeOperators(io.trino.type.BlockTypeOperators) Test(org.testng.annotations.Test)

Example 3 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class TestGroupedTopNRankBuilder method testMultiGroupTopN.

@Test(dataProvider = "produceRanking")
public void testMultiGroupTopN(boolean produceRanking) {
    TypeOperators typeOperators = new TypeOperators();
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(typeOperators);
    List<Type> types = ImmutableList.of(BIGINT, DOUBLE);
    GroupByHash groupByHash = createGroupByHash(ImmutableList.of(types.get(0)), ImmutableList.of(0), NOOP, typeOperators, blockTypeOperators);
    GroupedTopNBuilder groupedTopNBuilder = new GroupedTopNRankBuilder(types, new SimplePageWithPositionComparator(types, ImmutableList.of(1), ImmutableList.of(ASC_NULLS_LAST), typeOperators), new SimplePageWithPositionEqualsAndHash(types, ImmutableList.of(1), blockTypeOperators), 3, produceRanking, groupByHash);
    // Expected effect:
    // Group 0 [0.2 x 1 => rank=1, 0.3 x 3 => rank=2]
    // Group 1 [0.2 x 1 => rank=1]
    assertTrue(groupedTopNBuilder.processPage(rowPageBuilder(types).row(0L, 0.3).row(0L, 0.3).row(0L, 0.3).row(0L, 0.2).row(1L, 0.2).build()).process());
    // Page should be dropped, because all values too large to be considered
    assertTrue(groupedTopNBuilder.processPage(rowPageBuilder(types).row(0L, 0.4).row(1L, 0.4).build()).process());
    // Next page should cause evict 0.3 from group 0, which should cause the first page to be compacted
    // Expected effect:
    // Group 0 [0.1 x 1 => rank=1, 0.2 x 2 => rank=2]
    // Group 1 [0.2 x 2 => rank=1, 0.3 x 2 => rank=3]
    assertTrue(groupedTopNBuilder.processPage(rowPageBuilder(types).row(0L, 0.1).row(1L, 0.2).row(0L, 0.3).row(0L, 0.2).row(1L, 0.5).row(1L, 0.4).row(1L, 0.3).row(1L, 0.3).build()).process());
    List<Page> output = ImmutableList.copyOf(groupedTopNBuilder.buildResult());
    assertEquals(output.size(), 1);
    List<Type> outputTypes = ImmutableList.of(BIGINT, DOUBLE, BIGINT);
    Page expected = rowPageBuilder(outputTypes).row(0, 0.1, 1).row(0, 0.2, 2).row(0, 0.2, 2).row(1, 0.2, 1).row(1, 0.2, 1).row(1, 0.3, 3).row(1, 0.3, 3).build();
    if (!produceRanking) {
        outputTypes = outputTypes.subList(0, outputTypes.size() - 1);
        expected = dropLastColumn(expected);
    }
    assertPageEquals(outputTypes, getOnlyElement(output), expected);
}
Also used : Type(io.trino.spi.type.Type) BlockTypeOperators(io.trino.type.BlockTypeOperators) Page(io.trino.spi.Page) TypeOperators(io.trino.spi.type.TypeOperators) BlockTypeOperators(io.trino.type.BlockTypeOperators) Test(org.testng.annotations.Test)

Example 4 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class TestDynamicFilterSourceOperator method setUp.

@BeforeMethod
public void setUp() {
    blockTypeOperators = new BlockTypeOperators(new TypeOperators());
    executor = newCachedThreadPool(daemonThreadsNamed(getClass().getSimpleName() + "-%s"));
    scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed(getClass().getSimpleName() + "-scheduledExecutor-%s"));
    pipelineContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION).addPipelineContext(0, true, true, false);
    partitions = ImmutableList.builder();
}
Also used : BlockTypeOperators(io.trino.type.BlockTypeOperators) TypeOperators(io.trino.spi.type.TypeOperators) BlockTypeOperators(io.trino.type.BlockTypeOperators) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 5 with BlockTypeOperators

use of io.trino.type.BlockTypeOperators in project trino by trinodb.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    PageSourceManager pageSourceManager = new PageSourceManager();
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(PLANNER_CONTEXT.getTypeOperators());
    NodeScheduler nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, blockTypeOperators);
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(PLANNER_CONTEXT.getFunctionManager(), 0);
    return new LocalExecutionPlanner(PLANNER_CONTEXT, createTestingTypeAnalyzer(PLANNER_CONTEXT), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockDirectExchangeClientSupplier(), new ExpressionCompiler(PLANNER_CONTEXT.getFunctionManager(), pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(PLANNER_CONTEXT.getFunctionManager()), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new PagesIndex.TestingFactory(false), new JoinCompiler(PLANNER_CONTEXT.getTypeOperators()), new TrinoOperatorFactories(), new OrderingCompiler(PLANNER_CONTEXT.getTypeOperators()), new DynamicFilterConfig(), blockTypeOperators, new TableExecuteContextManager(), new ExchangeManagerRegistry(new ExchangeHandleResolver()));
}
Also used : PlanFragment(io.trino.sql.planner.PlanFragment) JoinFilterFunctionCompiler(io.trino.sql.gen.JoinFilterFunctionCompiler) ObjectMapperProvider(io.airlift.json.ObjectMapperProvider) CatalogName(io.trino.connector.CatalogName) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) FinalizerService(io.trino.util.FinalizerService) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) TEST_SESSION(io.trino.SessionTestUtils.TEST_SESSION) SINGLE_DISTRIBUTION(io.trino.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) TableScanNode(io.trino.sql.planner.plan.TableScanNode) ImmutableMap(com.google.common.collect.ImmutableMap) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) PageSinkManager(io.trino.split.PageSinkManager) PageFunctionCompiler(io.trino.sql.gen.PageFunctionCompiler) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) UniformNodeSelectorFactory(io.trino.execution.scheduler.UniformNodeSelectorFactory) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) SplitMonitor(io.trino.event.SplitMonitor) MockDirectExchangeClientSupplier(io.trino.execution.TestSqlTaskManager.MockDirectExchangeClientSupplier) List(java.util.List) BIGINT(io.trino.spi.type.BigintType.BIGINT) Split(io.trino.metadata.Split) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) Optional(java.util.Optional) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) TestingColumnHandle(io.trino.testing.TestingMetadata.TestingColumnHandle) TypeAnalyzer.createTestingTypeAnalyzer(io.trino.sql.planner.TypeAnalyzer.createTestingTypeAnalyzer) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) JoinCompiler(io.trino.sql.gen.JoinCompiler) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) Partitioning(io.trino.sql.planner.Partitioning) PartitioningScheme(io.trino.sql.planner.PartitioningScheme) StatsAndCosts(io.trino.cost.StatsAndCosts) ExpressionCompiler(io.trino.sql.gen.ExpressionCompiler) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) TestingSplit(io.trino.testing.TestingSplit) StageExecutionDescriptor.ungroupedExecution(io.trino.operator.StageExecutionDescriptor.ungroupedExecution) EventListenerConfig(io.trino.eventlistener.EventListenerConfig) IndexManager(io.trino.index.IndexManager) PagesIndex(io.trino.operator.PagesIndex) Symbol(io.trino.sql.planner.Symbol) BlockTypeOperators(io.trino.type.BlockTypeOperators) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PageSourceManager(io.trino.split.PageSourceManager) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) PLANNER_CONTEXT(io.trino.sql.planner.TestingPlannerContext.PLANNER_CONTEXT) EventListenerManager(io.trino.eventlistener.EventListenerManager) TEST_TABLE_HANDLE(io.trino.testing.TestingHandles.TEST_TABLE_HANDLE) OutputBuffers(io.trino.execution.buffer.OutputBuffers) SOURCE_DISTRIBUTION(io.trino.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) PageFunctionCompiler(io.trino.sql.gen.PageFunctionCompiler) MockDirectExchangeClientSupplier(io.trino.execution.TestSqlTaskManager.MockDirectExchangeClientSupplier) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) PagesIndex(io.trino.operator.PagesIndex) PageSourceManager(io.trino.split.PageSourceManager) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) BlockTypeOperators(io.trino.type.BlockTypeOperators) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PageSinkManager(io.trino.split.PageSinkManager) JoinCompiler(io.trino.sql.gen.JoinCompiler) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) UniformNodeSelectorFactory(io.trino.execution.scheduler.UniformNodeSelectorFactory) JoinFilterFunctionCompiler(io.trino.sql.gen.JoinFilterFunctionCompiler) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) IndexManager(io.trino.index.IndexManager) FinalizerService(io.trino.util.FinalizerService) ExpressionCompiler(io.trino.sql.gen.ExpressionCompiler)

Aggregations

BlockTypeOperators (io.trino.type.BlockTypeOperators)23 TypeOperators (io.trino.spi.type.TypeOperators)20 Type (io.trino.spi.type.Type)9 Test (org.testng.annotations.Test)9 Page (io.trino.spi.Page)6 BeforeMethod (org.testng.annotations.BeforeMethod)6 ImmutableList (com.google.common.collect.ImmutableList)5 JoinCompiler (io.trino.sql.gen.JoinCompiler)5 PlanNodeId (io.trino.sql.planner.plan.PlanNodeId)5 PagesIndex (io.trino.operator.PagesIndex)4 FeaturesConfig (io.trino.FeaturesConfig)3 NodeScheduler (io.trino.execution.scheduler.NodeScheduler)3 NodeSchedulerConfig (io.trino.execution.scheduler.NodeSchedulerConfig)3 UniformNodeSelectorFactory (io.trino.execution.scheduler.UniformNodeSelectorFactory)3 InMemoryNodeManager (io.trino.metadata.InMemoryNodeManager)3 Driver (io.trino.operator.Driver)3 GroupByHashPageIndexerFactory (io.trino.operator.GroupByHashPageIndexerFactory)3 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)2 NodeTaskMap (io.trino.execution.NodeTaskMap)2 DriverFactory (io.trino.operator.DriverFactory)2