use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.
the class TestOrcPageSourceMemoryTracking method testTableScanOperator.
@Test
public void testTableScanOperator() {
// Numbers used in assertions in this test may change when implementation is modified,
// feel free to change them if they break in the future
DriverContext driverContext = testPreparer.newDriverContext();
SourceOperator operator = testPreparer.newTableScanOperator(driverContext);
assertEquals(driverContext.getSystemMemoryUsage(), 0);
long memoryUsage = -1;
int totalRows = 0;
while (totalRows < 20000) {
assertFalse(operator.isFinished());
Page page = operator.getOutput();
assertNotNull(page);
page.getBlock(1);
totalRows += page.getPositionCount();
if (memoryUsage == -1) {
memoryUsage = driverContext.getSystemMemoryUsage();
assertBetweenInclusive(memoryUsage, 180000L, 469999L);
System.out.println(String.format(Locale.ENGLISH, "[TotalRows: %d] memUsage: %d", totalRows, driverContext.getSystemMemoryUsage()));
} else {
System.out.println(String.format(Locale.ENGLISH, "[TotalRows: %d] memUsage: %d", totalRows, driverContext.getSystemMemoryUsage()));
}
}
memoryUsage = -1;
while (totalRows < 40000) {
assertFalse(operator.isFinished());
Page page = operator.getOutput();
assertNotNull(page);
page.getBlock(1);
if (memoryUsage == -1) {
memoryUsage = driverContext.getSystemMemoryUsage();
assertBetweenInclusive(memoryUsage, 460000L, 469999L);
} else {
assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage);
}
totalRows += page.getPositionCount();
}
memoryUsage = -1;
while (totalRows < NUM_ROWS) {
assertFalse(operator.isFinished());
Page page = operator.getOutput();
assertNotNull(page);
page.getBlock(1);
if (memoryUsage == -1) {
memoryUsage = driverContext.getSystemMemoryUsage();
assertBetweenInclusive(memoryUsage, 360000L, 369999L);
} else {
assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage);
}
totalRows += page.getPositionCount();
}
assertFalse(operator.isFinished());
assertNull(operator.getOutput());
assertTrue(operator.isFinished());
assertEquals(driverContext.getSystemMemoryUsage(), 0);
}
use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.
the class SqlTaskExecution method enqueueDriverSplitRunner.
private synchronized void enqueueDriverSplitRunner(boolean forceRunSplit, List<DriverSplitRunner> runners) {
// schedule driver to be executed
List<ListenableFuture<?>> finishedFutures = taskExecutor.enqueueSplits(taskHandle, forceRunSplit, runners);
checkState(finishedFutures.size() == runners.size(), "Expected %s futures but got %s", runners.size(), finishedFutures.size());
// when driver completes, update state and fire events
for (int i = 0; i < finishedFutures.size(); i++) {
ListenableFuture<?> finishedFuture = finishedFutures.get(i);
final DriverSplitRunner splitRunner = runners.get(i);
// record new driver
status.incrementRemainingDriver(splitRunner.getLifespan());
Futures.addCallback(finishedFuture, new FutureCallback<Object>() {
@Override
public void onSuccess(Object result) {
try (SetThreadName ignored = new SetThreadName("Task-%s", taskId)) {
// record driver is finished
status.decrementRemainingDriver(splitRunner.getLifespan());
checkTaskCompletion();
splitMonitor.splitCompletedEvent(taskId, getDriverStats());
if (snapshotEnabled) {
if (splitRunner.partitionedSplit != null) {
boolean tryScheduling;
synchronized (SqlTaskExecution.this) {
// Drivers for splits with split lifecycle has a non-null split.
// When the driver has finished successfully, remove its split from in-progress list.
Set<ScheduledSplit> inProgress = inProgressSplits.get(splitRunner.partitionedSplit.getPlanNodeId());
inProgress.remove(splitRunner.partitionedSplit);
tryScheduling = inProgress.isEmpty() && !pendingSources.get(splitRunner.partitionedSplit.getPlanNodeId()).isEmpty();
}
if (tryScheduling) {
// There are pending sources that can be scheduled now.
// Call "addSources" instead of "updateSources" so that "checkTaskCompletion" is invoked as well
addSources(Collections.emptyList());
}
} else {
// Report finished drivers to snapshot manager, but only for non-source pipelines. Source pipeline operators don't store their states.
splitRunner.driver.reportFinishedDriver();
}
}
}
}
@Override
public void onFailure(Throwable cause) {
try (SetThreadName ignored = new SetThreadName("Task-%s", taskId)) {
taskStateMachine.failed(cause);
// record driver is finished
status.decrementRemainingDriver(splitRunner.getLifespan());
// fire failed event with cause
splitMonitor.splitFailedEvent(taskId, getDriverStats(), cause);
}
}
private DriverStats getDriverStats() {
DriverContext driverContext = splitRunner.getDriverContext();
DriverStats driverStats;
if (driverContext != null) {
driverStats = driverContext.getDriverStats();
} else {
// split runner did not start successfully
driverStats = new DriverStats();
}
return driverStats;
}
}, notificationExecutor);
}
}
use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.
the class TestSingleInputSnapshotState method testStaticConstructor.
@Test
public void testStaticConstructor() {
ScheduledExecutorService scheduler = newScheduledThreadPool(4);
DriverContext driverContext = createTaskContext(scheduler, scheduler, TEST_SNAPSHOT_SESSION).addPipelineContext(0, true, true, false).addDriverContext();
OperatorContext operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("planNodeId"), "test");
SingleInputSnapshotState singleInputSnapshotState = SingleInputSnapshotState.forOperator(mock(Operator.class), operatorContext);
singleInputSnapshotState.processPage(regularPage);
}
use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.
the class TestLocalExchangeSourceOperator method createOperator.
private LocalExchangeSourceOperator createOperator(LocalExchange.LocalExchangeFactory localExchangeFactory, int totalInputChannels, Session session, int driverId) {
if (pipelineContext == null) {
ScheduledExecutorService scheduler = newScheduledThreadPool(4, daemonThreadsNamed("test-%s"));
ScheduledExecutorService scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s"));
OperatorFactory operatorFactory = new LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory(0, new PlanNodeId("test"), localExchangeFactory, totalInputChannels);
pipelineContext = createTaskContext(scheduler, scheduledExecutor, session).addPipelineContext(0, true, true, false);
}
DriverContext driverContext = pipelineContext.addDriverContext(Lifespan.taskWide(), driverId);
OperatorFactory operatorFactory = new LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory(0, new PlanNodeId("test"), localExchangeFactory, totalInputChannels);
Operator operator = operatorFactory.createOperator(driverContext);
assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0);
return (LocalExchangeSourceOperator) operator;
}
use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.
the class HashJoinBenchmark method createDrivers.
/*
select orderkey, quantity, totalprice
from lineitem join orders using (orderkey)
*/
@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
if (probeDriverFactory == null) {
List<Type> ordersTypes = getColumnTypes("orders", "orderkey", "totalprice");
OperatorFactory ordersTableScan = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey", "totalprice");
JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(ordersTypes, ImmutableList.of(0, 1).stream().map(ordersTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(ordersTypes::get).collect(toImmutableList()), 1, requireNonNull(ImmutableMap.of(), "layout is null"), false, false));
HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), OptionalInt.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory());
DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext();
DriverFactory buildDriverFactory = new DriverFactory(0, false, false, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty(), UNGROUPED_EXECUTION);
List<Type> lineItemTypes = getColumnTypes("lineitem", "orderkey", "quantity");
OperatorFactory lineItemTableScan = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "orderkey", "quantity");
OperatorFactory joinOperator = LOOKUP_JOIN_OPERATORS.innerJoin(1, new PlanNodeId("test"), lookupSourceFactoryManager, lineItemTypes, Ints.asList(0), OptionalInt.empty(), Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory());
NullOutputOperatorFactory output = new NullOutputOperatorFactory(2, new PlanNodeId("test"));
this.probeDriverFactory = new DriverFactory(1, true, true, ImmutableList.of(lineItemTableScan, joinOperator, output), OptionalInt.empty(), UNGROUPED_EXECUTION);
Driver driver = buildDriverFactory.createDriver(driverContext);
Future<LookupSourceProvider> lookupSourceProvider = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide()).createLookupSourceProvider();
while (!lookupSourceProvider.isDone()) {
driver.process();
}
getFutureValue(lookupSourceProvider).close();
}
DriverContext driverContext = taskContext.addPipelineContext(1, true, true, false).addDriverContext();
Driver driver = probeDriverFactory.createDriver(driverContext);
return ImmutableList.of(driver);
}
Aggregations