use of io.trino.operator.Driver in project trino by trinodb.
the class AbstractOperatorBenchmark method execute.
protected Map<String, Long> execute(TaskContext taskContext) {
List<Driver> drivers = createDrivers(taskContext);
long peakMemory = 0;
boolean done = false;
while (!done) {
boolean processed = false;
for (Driver driver : drivers) {
if (!driver.isFinished()) {
driver.process();
long lastPeakMemory = peakMemory;
peakMemory = taskContext.getTaskStats().getUserMemoryReservation().toBytes();
if (peakMemory <= lastPeakMemory) {
peakMemory = lastPeakMemory;
}
processed = true;
}
}
done = !processed;
}
return ImmutableMap.of("peak_memory", peakMemory);
}
use of io.trino.operator.Driver in project trino by trinodb.
the class HashBuildBenchmark method createDrivers.
@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
// hash build
List<Type> ordersTypes = getColumnTypes("orders", "orderkey", "totalprice");
OperatorFactory ordersTableScan = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey", "totalprice");
BlockTypeOperators blockTypeOperators = new BlockTypeOperators(new TypeOperators());
JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(ordersTypes, ImmutableList.of(0, 1).stream().map(ordersTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(ordersTypes::get).collect(toImmutableList()), 1, false, blockTypeOperators));
HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), OptionalInt.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory(), incrementalLoadFactorHashArraySizeSupplier(session));
DriverFactory hashBuildDriverFactory = new DriverFactory(0, true, true, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty(), UNGROUPED_EXECUTION);
// empty join so build finishes
ImmutableList.Builder<OperatorFactory> joinDriversBuilder = ImmutableList.builder();
joinDriversBuilder.add(new ValuesOperatorFactory(0, new PlanNodeId("values"), ImmutableList.of()));
OperatorFactory joinOperator = operatorFactories.innerJoin(2, new PlanNodeId("test"), lookupSourceFactoryManager, false, false, false, ImmutableList.of(BIGINT), Ints.asList(0), OptionalInt.empty(), Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory(), blockTypeOperators);
joinDriversBuilder.add(joinOperator);
joinDriversBuilder.add(new NullOutputOperatorFactory(3, new PlanNodeId("test")));
DriverFactory joinDriverFactory = new DriverFactory(1, true, true, joinDriversBuilder.build(), OptionalInt.empty(), UNGROUPED_EXECUTION);
Driver hashBuildDriver = hashBuildDriverFactory.createDriver(taskContext.addPipelineContext(0, true, true, false).addDriverContext());
hashBuildDriverFactory.noMoreDrivers();
Driver joinDriver = joinDriverFactory.createDriver(taskContext.addPipelineContext(1, true, true, false).addDriverContext());
joinDriverFactory.noMoreDrivers();
return ImmutableList.of(hashBuildDriver, joinDriver);
}
use of io.trino.operator.Driver in project trino by trinodb.
the class HashJoinBenchmark method createDrivers.
/*
select orderkey, quantity, totalprice
from lineitem join orders using (orderkey)
*/
@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
if (probeDriverFactory == null) {
List<Type> ordersTypes = getColumnTypes("orders", "orderkey", "totalprice");
OperatorFactory ordersTableScan = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey", "totalprice");
BlockTypeOperators blockTypeOperators = new BlockTypeOperators(new TypeOperators());
JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(ordersTypes, ImmutableList.of(0, 1).stream().map(ordersTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(ordersTypes::get).collect(toImmutableList()), 1, false, blockTypeOperators));
HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), OptionalInt.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory(), incrementalLoadFactorHashArraySizeSupplier(session));
DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext();
DriverFactory buildDriverFactory = new DriverFactory(0, false, false, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty(), UNGROUPED_EXECUTION);
List<Type> lineItemTypes = getColumnTypes("lineitem", "orderkey", "quantity");
OperatorFactory lineItemTableScan = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "orderkey", "quantity");
OperatorFactory joinOperator = operatorFactories.innerJoin(1, new PlanNodeId("test"), lookupSourceFactoryManager, false, false, false, lineItemTypes, Ints.asList(0), OptionalInt.empty(), Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory(), blockTypeOperators);
NullOutputOperatorFactory output = new NullOutputOperatorFactory(2, new PlanNodeId("test"));
this.probeDriverFactory = new DriverFactory(1, true, true, ImmutableList.of(lineItemTableScan, joinOperator, output), OptionalInt.empty(), UNGROUPED_EXECUTION);
Driver driver = buildDriverFactory.createDriver(driverContext);
Future<LookupSourceProvider> lookupSourceProvider = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide()).createLookupSourceProvider();
while (!lookupSourceProvider.isDone()) {
driver.process();
}
getFutureValue(lookupSourceProvider).close();
}
DriverContext driverContext = taskContext.addPipelineContext(1, true, true, false).addDriverContext();
Driver driver = probeDriverFactory.createDriver(driverContext);
return ImmutableList.of(driver);
}
use of io.trino.operator.Driver in project trino by trinodb.
the class MemoryLocalQueryRunner method execute.
public List<Page> execute(@Language("SQL") String query) {
MemoryPool memoryPool = new MemoryPool(DataSize.of(2, GIGABYTE));
SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(DataSize.of(1, GIGABYTE));
QueryContext queryContext = new QueryContext(new QueryId("test"), DataSize.of(1, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), DataSize.of(4, GIGABYTE), spillSpaceTracker);
TaskContext taskContext = queryContext.addTaskContext(new TaskStateMachine(new TaskId(new StageId("query", 0), 0, 0), localQueryRunner.getExecutor()), localQueryRunner.getDefaultSession(), () -> {
}, false, false);
// Use NullOutputFactory to avoid coping out results to avoid affecting benchmark results
ImmutableList.Builder<Page> output = ImmutableList.builder();
List<Driver> drivers = localQueryRunner.createDrivers(query, new PageConsumerOperator.PageConsumerOutputFactory(types -> output::add), taskContext);
boolean done = false;
while (!done) {
boolean processed = false;
for (Driver driver : drivers) {
if (!driver.isFinished()) {
driver.process();
processed = true;
}
}
done = !processed;
}
return output.build();
}
Aggregations