Examples with DriverFactory io.prestosql.operator.DriverFactory used on opensource projects

Example 1 with DriverFactory

use of io.prestosql.operator.DriverFactory in project hetu-core by openlookeng.

the class LocalExecutionPlanner method calculateTotalCountOfTaskComponentToBeCaptured.

private static int calculateTotalCountOfTaskComponentToBeCaptured(TaskContext taskContext, LocalExecutionPlanContext context, OutputBuffer outputBuffer) {
    int totalCount = 0;
    boolean outputPipelineIsTableScan = false;
    for (DriverFactory df : context.getDriverFactories()) {
        // Don't include operators in table-scan pipelines
        if (!isTableScanPipeline(df)) {
            if (isOuterJoinFromTableScanPipeline(df, context)) {
                // See Gitee issue Checkpoint - handle LookupOuterOperator pipelines
                // https://gitee.com/open_lookeng/dashboard/issues?id=I2LMIW
                // For outer pipelines forked from table-scan, only count the lookup-outer operator
                totalCount += df.getDriverInstances().orElse(1);
            } else {
                totalCount += df.getDriverInstances().orElse(1) * df.getOperatorFactories().size();
            }
        } else if (df.isOutputDriver()) {
            outputPipelineIsTableScan = true;
        }
    }
    int stageId = taskContext.getTaskId().getStageId().getId();
    // If output pipeline is not also source pipeline, then OutputBuffer state needs to be captured
    if (stageId > 0 && !outputPipelineIsTableScan) {
        // For partitioned output buffer, each partition has its own snapshot state, so need to count all of them.
        // For other output buffers, there is a single snapshot state, so use 1.
        OutputBufferInfo info = outputBuffer.getInfo();
        if (info.getType().equals("PARTITIONED")) {
            totalCount += info.getBuffers().size();
        } else {
            totalCount++;
        }
    }
    return totalCount;
}

Example 2 with DriverFactory

use of io.prestosql.operator.DriverFactory in project hetu-core by openlookeng.

the class HashBuildBenchmark method createDrivers.

@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
    // hash build
    List<Type> ordersTypes = getColumnTypes("orders", "orderkey", "totalprice");
    OperatorFactory ordersTableScan = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey", "totalprice");
    JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(ordersTypes, ImmutableList.of(0, 1).stream().map(ordersTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(ordersTypes::get).collect(toImmutableList()), 1, requireNonNull(ImmutableMap.of(), "layout is null"), false, false));
    HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), OptionalInt.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory());
    DriverFactory hashBuildDriverFactory = new DriverFactory(0, true, true, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty(), UNGROUPED_EXECUTION);
    // empty join so build finishes
    ImmutableList.Builder<OperatorFactory> joinDriversBuilder = ImmutableList.builder();
    joinDriversBuilder.add(new ValuesOperatorFactory(0, new PlanNodeId("values"), ImmutableList.of()));
    OperatorFactory joinOperator = LOOKUP_JOIN_OPERATORS.innerJoin(2, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(BIGINT), Ints.asList(0), OptionalInt.empty(), Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory());
    joinDriversBuilder.add(joinOperator);
    joinDriversBuilder.add(new NullOutputOperatorFactory(3, new PlanNodeId("test")));
    DriverFactory joinDriverFactory = new DriverFactory(1, true, true, joinDriversBuilder.build(), OptionalInt.empty(), UNGROUPED_EXECUTION);
    Driver hashBuildDriver = hashBuildDriverFactory.createDriver(taskContext.addPipelineContext(0, true, true, false).addDriverContext());
    hashBuildDriverFactory.noMoreDrivers();
    Driver joinDriver = joinDriverFactory.createDriver(taskContext.addPipelineContext(1, true, true, false).addDriverContext());
    joinDriverFactory.noMoreDrivers();
    return ImmutableList.of(hashBuildDriver, joinDriver);
}

Example 3 with DriverFactory

use of io.prestosql.operator.DriverFactory in project hetu-core by openlookeng.

the class HashJoinBenchmark method createDrivers.

/*
    select orderkey, quantity, totalprice
    from lineitem join orders using (orderkey)
     */
@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
    if (probeDriverFactory == null) {
        List<Type> ordersTypes = getColumnTypes("orders", "orderkey", "totalprice");
        OperatorFactory ordersTableScan = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey", "totalprice");
        JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(ordersTypes, ImmutableList.of(0, 1).stream().map(ordersTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(ordersTypes::get).collect(toImmutableList()), 1, requireNonNull(ImmutableMap.of(), "layout is null"), false, false));
        HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), OptionalInt.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory());
        DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext();
        DriverFactory buildDriverFactory = new DriverFactory(0, false, false, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty(), UNGROUPED_EXECUTION);
        List<Type> lineItemTypes = getColumnTypes("lineitem", "orderkey", "quantity");
        OperatorFactory lineItemTableScan = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "orderkey", "quantity");
        OperatorFactory joinOperator = LOOKUP_JOIN_OPERATORS.innerJoin(1, new PlanNodeId("test"), lookupSourceFactoryManager, lineItemTypes, Ints.asList(0), OptionalInt.empty(), Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory());
        NullOutputOperatorFactory output = new NullOutputOperatorFactory(2, new PlanNodeId("test"));
        this.probeDriverFactory = new DriverFactory(1, true, true, ImmutableList.of(lineItemTableScan, joinOperator, output), OptionalInt.empty(), UNGROUPED_EXECUTION);
        Driver driver = buildDriverFactory.createDriver(driverContext);
        Future<LookupSourceProvider> lookupSourceProvider = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide()).createLookupSourceProvider();
        while (!lookupSourceProvider.isDone()) {
            driver.process();
        }
        getFutureValue(lookupSourceProvider).close();
    }
    DriverContext driverContext = taskContext.addPipelineContext(1, true, true, false).addDriverContext();
    Driver driver = probeDriverFactory.createDriver(driverContext);
    return ImmutableList.of(driver);
}

Example 4 with DriverFactory

use of io.prestosql.operator.DriverFactory in project hetu-core by openlookeng.

the class AbstractSimpleOperatorBenchmark method createDriverFactory.

protected DriverFactory createDriverFactory() {
    List<OperatorFactory> operatorFactories = new ArrayList<>(createOperatorFactories());
    operatorFactories.add(new NullOutputOperatorFactory(999, new PlanNodeId("test")));
    return new DriverFactory(0, true, true, operatorFactories, OptionalInt.empty(), UNGROUPED_EXECUTION);
}

Example 5 with DriverFactory

use of io.prestosql.operator.DriverFactory in project hetu-core by openlookeng.

the class IndexLoader method streamIndexDataForSingleKey.

public IndexedData streamIndexDataForSingleKey(UpdateRequest updateRequest) {
    Page indexKeyTuple = updateRequest.getPage().getRegion(0, 1);
    PageBuffer pageBuffer = new PageBuffer(100);
    DriverFactory driverFactory = indexBuildDriverFactoryProvider.createStreaming(pageBuffer, indexKeyTuple);
    Driver driver = driverFactory.createDriver(pipelineContext.addDriverContext());
    PageRecordSet pageRecordSet = new PageRecordSet(keyTypes, indexKeyTuple);
    PlanNodeId planNodeId = driverFactory.getSourceId().get();
    ScheduledSplit split = new ScheduledSplit(0, planNodeId, new Split(INDEX_CONNECTOR_ID, new IndexSplit(pageRecordSet), Lifespan.taskWide()));
    driver.updateSource(new TaskSource(planNodeId, ImmutableSet.of(split), true));
    return new StreamingIndexedData(outputTypes, keyTypes, indexKeyTuple, pageBuffer, driver);
}