use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.
the class TaskTestUtils method createTestingPlanner.
public static LocalExecutionPlanner createTestingPlanner() {
Metadata metadata = createTestMetadataManager();
PageSourceManager pageSourceManager = new PageSourceManager();
HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
FeaturesConfig featuresConfig = new FeaturesConfig();
CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
// we don't start the finalizer so nothing will be collected, which is ok for a test
FinalizerService finalizerService = new FinalizerService();
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
NodeInfo nodeInfo = new NodeInfo("test");
FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}), (types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, (types, partitionFunction, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}
use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.
the class TestScanFilterAndProjectOperator method testPageYield.
@Test
public void testPageYield() {
int totalRows = 1000;
Page input = SequencePageBuilder.createSequencePage(ImmutableList.of(BIGINT), totalRows, 1);
DriverContext driverContext = newDriverContext();
// 20 columns; each column is associated with a function that will force yield per projection
int totalColumns = 20;
ImmutableList.Builder<SqlScalarFunction> functions = ImmutableList.builder();
for (int i = 0; i < totalColumns; i++) {
functions.add(new GenericLongFunction("page_col" + i, value -> {
driverContext.getYieldSignal().forceYieldForTesting();
return value;
}));
}
Metadata localMetadata = functionAssertions.getMetadata();
localMetadata.getFunctionAndTypeManager().registerBuiltInFunctions(functions.build());
// match each column with a projection
ExpressionCompiler compiler = new ExpressionCompiler(localMetadata, new PageFunctionCompiler(localMetadata, 0));
ImmutableList.Builder<RowExpression> projections = ImmutableList.builder();
for (int i = 0; i < totalColumns; i++) {
projections.add(call(QualifiedObjectName.valueOfDefaultFunction("generic_long_page_col" + i).toString(), new BuiltInFunctionHandle(internalScalarFunction(QualifiedObjectName.valueOfDefaultFunction("generic_long_page_col" + i), BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature()))), BIGINT, field(0, BIGINT)));
}
Supplier<CursorProcessor> cursorProcessor = compiler.compileCursorProcessor(Optional.empty(), projections.build(), "key");
Supplier<PageProcessor> pageProcessor = compiler.compilePageProcessor(Optional.empty(), projections.build(), MAX_BATCH_SIZE);
ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory factory = new ScanFilterAndProjectOperator.ScanFilterAndProjectOperatorFactory(0, new PlanNodeId("test"), new PlanNodeId("0"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(ImmutableList.of(input)), cursorProcessor, pageProcessor, TEST_TABLE_HANDLE, ImmutableList.of(), null, ImmutableList.of(BIGINT), new DataSize(0, BYTE), 0, ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), false, Optional.empty(), 0, 0);
SourceOperator operator = factory.createOperator(driverContext);
operator.addSplit(new Split(new CatalogName("test"), TestingSplit.createLocalSplit(), Lifespan.taskWide()));
operator.noMoreSplits();
// exactly 20 blocks (one for each column) and the PageProcessor will be able to create a Page out of it.
for (int i = 1; i <= totalRows * totalColumns; i++) {
driverContext.getYieldSignal().setWithDelay(SECONDS.toNanos(1000), driverContext.getYieldExecutor());
Page page = operator.getOutput();
if (i == totalColumns) {
assertNotNull(page);
assertEquals(page.getPositionCount(), totalRows);
assertEquals(page.getChannelCount(), totalColumns);
for (int j = 0; j < totalColumns; j++) {
assertEquals(toValues(BIGINT, page.getBlock(j)), toValues(BIGINT, input.getBlock(0)));
}
} else {
assertNull(page);
}
driverContext.getYieldSignal().reset();
}
}
use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.
the class TestPageProcessor method testExpressionProfiler.
@Test
public void testExpressionProfiler() {
CallExpression add10Expression = call(ADD.getFunctionName().toString(), new BuiltInFunctionHandle(internalOperator(ADD, BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature(), BIGINT.getTypeSignature()))), BIGINT, field(0, BIGINT), constant(10L, BIGINT));
TestingTicker testingTicker = new TestingTicker();
PageFunctionCompiler functionCompiler = new PageFunctionCompiler(createTestMetadataManager(), 0);
Supplier<PageProjection> projectionSupplier = functionCompiler.compileProjection(add10Expression, Optional.empty());
PageProjection projection = projectionSupplier.get();
Page page = new Page(createLongSequenceBlock(1, 11));
ExpressionProfiler profiler = new ExpressionProfiler(testingTicker, SPLIT_RUN_QUANTA);
for (int i = 0; i < 100; i++) {
profiler.start();
Work<Block> work = projection.project(SESSION, new DriverYieldSignal(), page, SelectedPositions.positionsRange(0, page.getPositionCount()));
if (i < 10) {
// increment the ticker with a large value to mark the expression as expensive
testingTicker.increment(10, SECONDS);
profiler.stop(page.getPositionCount());
assertTrue(profiler.isExpressionExpensive());
} else {
testingTicker.increment(0, NANOSECONDS);
profiler.stop(page.getPositionCount());
assertFalse(profiler.isExpressionExpensive());
}
work.process();
}
}
use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.
the class TestPageProcessorCompiler method setup.
@BeforeClass
public void setup() {
metadata = createTestMetadataManager();
compiler = new ExpressionCompiler(metadata, new PageFunctionCompiler(metadata, 0));
}
use of io.prestosql.sql.gen.PageFunctionCompiler in project hetu-core by openlookeng.
the class PredicateFilterBenchmark method createOperatorFactories.
@Override
protected List<? extends OperatorFactory> createOperatorFactories() {
OperatorFactory tableScanOperator = createTableScanOperator(0, new PlanNodeId("test"), "orders", "totalprice");
RowExpression filter = call(GREATER_THAN_OR_EQUAL.name(), localQueryRunner.getMetadata().getFunctionAndTypeManager().resolveOperatorFunctionHandle(GREATER_THAN_OR_EQUAL, fromTypes(DOUBLE, DOUBLE)), BOOLEAN, field(0, DOUBLE), constant(50000.0, DOUBLE));
ExpressionCompiler expressionCompiler = new ExpressionCompiler(localQueryRunner.getMetadata(), new PageFunctionCompiler(localQueryRunner.getMetadata(), 0));
Supplier<PageProcessor> pageProcessor = expressionCompiler.compilePageProcessor(Optional.of(filter), ImmutableList.of(field(0, DOUBLE)));
FilterAndProjectOperator.FilterAndProjectOperatorFactory filterAndProjectOperator = new FilterAndProjectOperator.FilterAndProjectOperatorFactory(1, new PlanNodeId("test"), pageProcessor, ImmutableList.of(DOUBLE), new DataSize(0, BYTE), 0);
return ImmutableList.of(tableScanOperator, filterAndProjectOperator);
}
Aggregations