Search in sources :

Example 1 with DriverContext

use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.

the class TestOrcPageSourceMemoryTracking method testTableScanOperator.

@Test
public void testTableScanOperator() {
    // Numbers used in assertions in this test may change when implementation is modified,
    // feel free to change them if they break in the future
    DriverContext driverContext = testPreparer.newDriverContext();
    SourceOperator operator = testPreparer.newTableScanOperator(driverContext);
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
    long memoryUsage = -1;
    int totalRows = 0;
    while (totalRows < 20000) {
        assertFalse(operator.isFinished());
        Page page = operator.getOutput();
        assertNotNull(page);
        page.getBlock(1);
        totalRows += page.getPositionCount();
        if (memoryUsage == -1) {
            memoryUsage = driverContext.getSystemMemoryUsage();
            assertBetweenInclusive(memoryUsage, 180000L, 469999L);
            System.out.println(String.format(Locale.ENGLISH, "[TotalRows: %d] memUsage: %d", totalRows, driverContext.getSystemMemoryUsage()));
        } else {
            System.out.println(String.format(Locale.ENGLISH, "[TotalRows: %d] memUsage: %d", totalRows, driverContext.getSystemMemoryUsage()));
        }
    }
    memoryUsage = -1;
    while (totalRows < 40000) {
        assertFalse(operator.isFinished());
        Page page = operator.getOutput();
        assertNotNull(page);
        page.getBlock(1);
        if (memoryUsage == -1) {
            memoryUsage = driverContext.getSystemMemoryUsage();
            assertBetweenInclusive(memoryUsage, 460000L, 469999L);
        } else {
            assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage);
        }
        totalRows += page.getPositionCount();
    }
    memoryUsage = -1;
    while (totalRows < NUM_ROWS) {
        assertFalse(operator.isFinished());
        Page page = operator.getOutput();
        assertNotNull(page);
        page.getBlock(1);
        if (memoryUsage == -1) {
            memoryUsage = driverContext.getSystemMemoryUsage();
            assertBetweenInclusive(memoryUsage, 360000L, 369999L);
        } else {
            assertEquals(driverContext.getSystemMemoryUsage(), memoryUsage);
        }
        totalRows += page.getPositionCount();
    }
    assertFalse(operator.isFinished());
    assertNull(operator.getOutput());
    assertTrue(operator.isFinished());
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
}
Also used : DriverContext(io.prestosql.operator.DriverContext) SourceOperator(io.prestosql.operator.SourceOperator) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 2 with DriverContext

use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.

the class SqlTaskExecution method enqueueDriverSplitRunner.

private synchronized void enqueueDriverSplitRunner(boolean forceRunSplit, List<DriverSplitRunner> runners) {
    // schedule driver to be executed
    List<ListenableFuture<?>> finishedFutures = taskExecutor.enqueueSplits(taskHandle, forceRunSplit, runners);
    checkState(finishedFutures.size() == runners.size(), "Expected %s futures but got %s", runners.size(), finishedFutures.size());
    // when driver completes, update state and fire events
    for (int i = 0; i < finishedFutures.size(); i++) {
        ListenableFuture<?> finishedFuture = finishedFutures.get(i);
        final DriverSplitRunner splitRunner = runners.get(i);
        // record new driver
        status.incrementRemainingDriver(splitRunner.getLifespan());
        Futures.addCallback(finishedFuture, new FutureCallback<Object>() {

            @Override
            public void onSuccess(Object result) {
                try (SetThreadName ignored = new SetThreadName("Task-%s", taskId)) {
                    // record driver is finished
                    status.decrementRemainingDriver(splitRunner.getLifespan());
                    checkTaskCompletion();
                    splitMonitor.splitCompletedEvent(taskId, getDriverStats());
                    if (snapshotEnabled) {
                        if (splitRunner.partitionedSplit != null) {
                            boolean tryScheduling;
                            synchronized (SqlTaskExecution.this) {
                                // Drivers for splits with split lifecycle has a non-null split.
                                // When the driver has finished successfully, remove its split from in-progress list.
                                Set<ScheduledSplit> inProgress = inProgressSplits.get(splitRunner.partitionedSplit.getPlanNodeId());
                                inProgress.remove(splitRunner.partitionedSplit);
                                tryScheduling = inProgress.isEmpty() && !pendingSources.get(splitRunner.partitionedSplit.getPlanNodeId()).isEmpty();
                            }
                            if (tryScheduling) {
                                // There are pending sources that can be scheduled now.
                                // Call "addSources" instead of "updateSources" so that "checkTaskCompletion" is invoked as well
                                addSources(Collections.emptyList());
                            }
                        } else {
                            // Report finished drivers to snapshot manager, but only for non-source pipelines. Source pipeline operators don't store their states.
                            splitRunner.driver.reportFinishedDriver();
                        }
                    }
                }
            }

            @Override
            public void onFailure(Throwable cause) {
                try (SetThreadName ignored = new SetThreadName("Task-%s", taskId)) {
                    taskStateMachine.failed(cause);
                    // record driver is finished
                    status.decrementRemainingDriver(splitRunner.getLifespan());
                    // fire failed event with cause
                    splitMonitor.splitFailedEvent(taskId, getDriverStats(), cause);
                }
            }

            private DriverStats getDriverStats() {
                DriverContext driverContext = splitRunner.getDriverContext();
                DriverStats driverStats;
                if (driverContext != null) {
                    driverStats = driverContext.getDriverStats();
                } else {
                    // split runner did not start successfully
                    driverStats = new DriverStats();
                }
                return driverStats;
            }
        }, notificationExecutor);
    }
}
Also used : DriverContext(io.prestosql.operator.DriverContext) DriverStats(io.prestosql.operator.DriverStats) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) HashSet(java.util.HashSet) SetThreadName(io.airlift.concurrent.SetThreadName) ListenableFuture(com.google.common.util.concurrent.ListenableFuture)

Example 3 with DriverContext

use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.

the class TestSingleInputSnapshotState method testStaticConstructor.

@Test
public void testStaticConstructor() {
    ScheduledExecutorService scheduler = newScheduledThreadPool(4);
    DriverContext driverContext = createTaskContext(scheduler, scheduler, TEST_SNAPSHOT_SESSION).addPipelineContext(0, true, true, false).addDriverContext();
    OperatorContext operatorContext = driverContext.addOperatorContext(1, new PlanNodeId("planNodeId"), "test");
    SingleInputSnapshotState singleInputSnapshotState = SingleInputSnapshotState.forOperator(mock(Operator.class), operatorContext);
    singleInputSnapshotState.processPage(regularPage);
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Operator(io.prestosql.operator.Operator) DriverContext(io.prestosql.operator.DriverContext) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) OperatorContext(io.prestosql.operator.OperatorContext) Test(org.testng.annotations.Test)

Example 4 with DriverContext

use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.

the class TestLocalExchangeSourceOperator method createOperator.

private LocalExchangeSourceOperator createOperator(LocalExchange.LocalExchangeFactory localExchangeFactory, int totalInputChannels, Session session, int driverId) {
    if (pipelineContext == null) {
        ScheduledExecutorService scheduler = newScheduledThreadPool(4, daemonThreadsNamed("test-%s"));
        ScheduledExecutorService scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s"));
        OperatorFactory operatorFactory = new LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory(0, new PlanNodeId("test"), localExchangeFactory, totalInputChannels);
        pipelineContext = createTaskContext(scheduler, scheduledExecutor, session).addPipelineContext(0, true, true, false);
    }
    DriverContext driverContext = pipelineContext.addDriverContext(Lifespan.taskWide(), driverId);
    OperatorFactory operatorFactory = new LocalExchangeSourceOperator.LocalExchangeSourceOperatorFactory(0, new PlanNodeId("test"), localExchangeFactory, totalInputChannels);
    Operator operator = operatorFactory.createOperator(driverContext);
    assertEquals(operator.getOperatorContext().getOperatorStats().getSystemMemoryReservation().toBytes(), 0);
    return (LocalExchangeSourceOperator) operator;
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Operator(io.prestosql.operator.Operator) DriverContext(io.prestosql.operator.DriverContext) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) OperatorFactory(io.prestosql.operator.OperatorFactory)

Example 5 with DriverContext

use of io.prestosql.operator.DriverContext in project hetu-core by openlookeng.

the class HashJoinBenchmark method createDrivers.

/*
    select orderkey, quantity, totalprice
    from lineitem join orders using (orderkey)
     */
@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
    if (probeDriverFactory == null) {
        List<Type> ordersTypes = getColumnTypes("orders", "orderkey", "totalprice");
        OperatorFactory ordersTableScan = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey", "totalprice");
        JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(ordersTypes, ImmutableList.of(0, 1).stream().map(ordersTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(ordersTypes::get).collect(toImmutableList()), 1, requireNonNull(ImmutableMap.of(), "layout is null"), false, false));
        HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), OptionalInt.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory());
        DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext();
        DriverFactory buildDriverFactory = new DriverFactory(0, false, false, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty(), UNGROUPED_EXECUTION);
        List<Type> lineItemTypes = getColumnTypes("lineitem", "orderkey", "quantity");
        OperatorFactory lineItemTableScan = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "orderkey", "quantity");
        OperatorFactory joinOperator = LOOKUP_JOIN_OPERATORS.innerJoin(1, new PlanNodeId("test"), lookupSourceFactoryManager, lineItemTypes, Ints.asList(0), OptionalInt.empty(), Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory());
        NullOutputOperatorFactory output = new NullOutputOperatorFactory(2, new PlanNodeId("test"));
        this.probeDriverFactory = new DriverFactory(1, true, true, ImmutableList.of(lineItemTableScan, joinOperator, output), OptionalInt.empty(), UNGROUPED_EXECUTION);
        Driver driver = buildDriverFactory.createDriver(driverContext);
        Future<LookupSourceProvider> lookupSourceProvider = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide()).createLookupSourceProvider();
        while (!lookupSourceProvider.isDone()) {
            driver.process();
        }
        getFutureValue(lookupSourceProvider).close();
    }
    DriverContext driverContext = taskContext.addPipelineContext(1, true, true, false).addDriverContext();
    Driver driver = probeDriverFactory.createDriver(driverContext);
    return ImmutableList.of(driver);
}
Also used : DriverContext(io.prestosql.operator.DriverContext) HashBuilderOperatorFactory(io.prestosql.operator.HashBuilderOperator.HashBuilderOperatorFactory) Driver(io.prestosql.operator.Driver) PagesIndex(io.prestosql.operator.PagesIndex) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Type(io.prestosql.spi.type.Type) HashBuilderOperatorFactory(io.prestosql.operator.HashBuilderOperator.HashBuilderOperatorFactory) OperatorFactory(io.prestosql.operator.OperatorFactory) NullOutputOperatorFactory(io.prestosql.testing.NullOutputOperator.NullOutputOperatorFactory) DriverFactory(io.prestosql.operator.DriverFactory) PartitionedLookupSourceFactory(io.prestosql.operator.PartitionedLookupSourceFactory) NullOutputOperatorFactory(io.prestosql.testing.NullOutputOperator.NullOutputOperatorFactory) LookupSourceProvider(io.prestosql.operator.LookupSourceProvider)

Aggregations

DriverContext (io.prestosql.operator.DriverContext)24 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)16 Test (org.testng.annotations.Test)12 OperatorFactory (io.prestosql.operator.OperatorFactory)10 TaskContext (io.prestosql.operator.TaskContext)10 OperatorContext (io.prestosql.operator.OperatorContext)7 PagesSpatialIndexFactory (io.prestosql.operator.PagesSpatialIndexFactory)7 SpatialIndexBuilderOperatorFactory (io.prestosql.operator.SpatialIndexBuilderOperator.SpatialIndexBuilderOperatorFactory)7 SpatialJoinOperatorFactory (io.prestosql.operator.SpatialJoinOperator.SpatialJoinOperatorFactory)7 RowPagesBuilder (io.prestosql.RowPagesBuilder)5 Driver (io.prestosql.operator.Driver)5 Operator (io.prestosql.operator.Operator)5 OperatorAssertion.toMaterializedResult (io.prestosql.operator.OperatorAssertion.toMaterializedResult)5 MaterializedResult (io.prestosql.testing.MaterializedResult)5 TestingTaskContext (io.prestosql.testing.TestingTaskContext)5 PipelineContext (io.prestosql.operator.PipelineContext)4 Page (io.prestosql.spi.Page)4 ScheduledExecutorService (java.util.concurrent.ScheduledExecutorService)4 ImmutableList (com.google.common.collect.ImmutableList)3 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)3