Examples with JoinCompiler com.facebook.presto.sql.gen.JoinCompiler used on opensource projects

Example 1 with JoinCompiler

use of com.facebook.presto.sql.gen.JoinCompiler in project presto by prestodb.

the class LocalQueryRunner method createDrivers.

public List<Driver> createDrivers(Session session, @Language("SQL") String sql, OutputFactory outputFactory, TaskContext taskContext) {
    Plan plan = createPlan(session, sql);
    if (printPlan) {
        System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata, session));
    }
    SubPlan subplan = PlanFragmenter.createSubPlans(session, metadata, plan);
    if (!subplan.getChildren().isEmpty()) {
        throw new AssertionError("Expected subplan to have no children");
    }
    LocalExecutionPlanner executionPlanner = new LocalExecutionPlanner(metadata, sqlParser, Optional.empty(), pageSourceManager, indexManager, nodePartitioningManager, pageSinkManager, null, expressionCompiler, joinFilterFunctionCompiler, new IndexJoinLookupStats(), // make sure tests fail if compiler breaks
    new CompilerConfig().setInterpreterEnabled(false), new TaskManagerConfig().setTaskConcurrency(4), spillerFactory, blockEncodingSerde, new PagesIndex.TestingFactory(), new JoinCompiler(), new LookupJoinOperators(new JoinProbeCompiler()));
    // plan query
    LocalExecutionPlan localExecutionPlan = executionPlanner.plan(session, subplan.getFragment().getRoot(), subplan.getFragment().getPartitioningScheme().getOutputLayout(), plan.getTypes(), outputFactory);
    // generate sources
    List<TaskSource> sources = new ArrayList<>();
    long sequenceId = 0;
    for (TableScanNode tableScan : findTableScanNodes(subplan.getFragment().getRoot())) {
        TableLayoutHandle layout = tableScan.getLayout().get();
        SplitSource splitSource = splitManager.getSplits(session, layout);
        ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
        while (!splitSource.isFinished()) {
            for (Split split : getFutureValue(splitSource.getNextBatch(1000))) {
                scheduledSplits.add(new ScheduledSplit(sequenceId++, tableScan.getId(), split));
            }
        }
        sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
    }
    // create drivers
    List<Driver> drivers = new ArrayList<>();
    Map<PlanNodeId, DriverFactory> driverFactoriesBySource = new HashMap<>();
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
        for (int i = 0; i < driverFactory.getDriverInstances().orElse(1); i++) {
            if (driverFactory.getSourceId().isPresent()) {
                checkState(driverFactoriesBySource.put(driverFactory.getSourceId().get(), driverFactory) == null);
            } else {
                DriverContext driverContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver()).addDriverContext();
                Driver driver = driverFactory.createDriver(driverContext);
                drivers.add(driver);
            }
        }
    }
    // add sources to the drivers
    for (TaskSource source : sources) {
        DriverFactory driverFactory = driverFactoriesBySource.get(source.getPlanNodeId());
        checkState(driverFactory != null);
        for (ScheduledSplit split : source.getSplits()) {
            DriverContext driverContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver()).addDriverContext();
            Driver driver = driverFactory.createDriver(driverContext);
            driver.updateSource(new TaskSource(split.getPlanNodeId(), ImmutableSet.of(split), true));
            drivers.add(driver);
        }
    }
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
        driverFactory.close();
    }
    return ImmutableList.copyOf(drivers);
}

Example 2 with JoinCompiler

use of com.facebook.presto.sql.gen.JoinCompiler in project presto by prestodb.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    MetadataManager metadata = MetadataManager.createTestMetadataManager();
    PageSourceManager pageSourceManager = new PageSourceManager();
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
    return new LocalExecutionPlanner(metadata, new SqlParser(), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata), new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new CompilerConfig(), new TaskManagerConfig(), new BinarySpillerFactory(new BlockEncodingManager(metadata.getTypeManager()), new FeaturesConfig()), new TestingBlockEncodingSerde(new TestingTypeManager()), new PagesIndex.TestingFactory(), new JoinCompiler(), new LookupJoinOperators(new JoinProbeCompiler()));
}

Example 3 with JoinCompiler

use of com.facebook.presto.sql.gen.JoinCompiler in project presto by prestodb.

the class TestHashSemiJoinOperator method testProbeAndBuildNulls.

//Disabled till #6622 is fixed
@Test(dataProvider = "hashEnabledValues", enabled = false)
public void testProbeAndBuildNulls(boolean hashEnabled) throws Exception {
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true).addDriverContext();
    // build
    OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName());
    List<Type> buildTypes = ImmutableList.of(BIGINT);
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), buildTypes);
    Operator buildOperator = new ValuesOperator(operatorContext, buildTypes, rowPagesBuilder.row(0L).row(1L).row((Object) null).row(3L).build());
    SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory(1, new PlanNodeId("test"), buildOperator.getTypes().get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler());
    Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
    Driver driver = new Driver(driverContext, buildOperator, setBuilderOperator);
    while (!driver.isFinished()) {
        driver.process();
    }
    // probe
    List<Type> probeTypes = ImmutableList.of(BIGINT);
    RowPagesBuilder rowPagesBuilderProbe = rowPagesBuilder(hashEnabled, Ints.asList(0), probeTypes);
    List<Page> probeInput = rowPagesBuilderProbe.row(0L).row((Object) null).row(1L).row(2L).build();
    HashSemiJoinOperatorFactory joinOperatorFactory = new HashSemiJoinOperatorFactory(2, new PlanNodeId("test"), setBuilderOperatorFactory.getSetProvider(), rowPagesBuilderProbe.getTypes(), 0);
    // expected
    MaterializedResult expected = resultBuilder(driverContext.getSession(), concat(probeTypes, ImmutableList.of(BOOLEAN))).row(0L, true).row(null, null).row(1L, true).row(2L, null).build();
    OperatorAssertion.assertOperatorEquals(joinOperatorFactory, driverContext, probeInput, expected, hashEnabled, ImmutableList.of(probeTypes.size()));
}

Example 4 with JoinCompiler

use of com.facebook.presto.sql.gen.JoinCompiler in project presto by prestodb.

the class TestHashSemiJoinOperator method testSemiJoin.

@Test(dataProvider = "hashEnabledValues")
public void testSemiJoin(boolean hashEnabled) throws Exception {
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true).addDriverContext();
    // build
    OperatorContext operatorContext = driverContext.addOperatorContext(0, new PlanNodeId("test"), ValuesOperator.class.getSimpleName());
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
    Operator buildOperator = new ValuesOperator(operatorContext, rowPagesBuilder.getTypes(), rowPagesBuilder.row(10L).row(30L).row(30L).row(35L).row(36L).row(37L).row(50L).build());
    SetBuilderOperatorFactory setBuilderOperatorFactory = new SetBuilderOperatorFactory(1, new PlanNodeId("test"), buildOperator.getTypes().get(0), 0, rowPagesBuilder.getHashChannel(), 10, new JoinCompiler());
    Operator setBuilderOperator = setBuilderOperatorFactory.createOperator(driverContext);
    Driver driver = new Driver(driverContext, buildOperator, setBuilderOperator);
    while (!driver.isFinished()) {
        driver.process();
    }
    // probe
    List<Type> probeTypes = ImmutableList.of(BIGINT, BIGINT);
    RowPagesBuilder rowPagesBuilderProbe = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT, BIGINT);
    List<Page> probeInput = rowPagesBuilderProbe.addSequencePage(10, 30, 0).build();
    HashSemiJoinOperatorFactory joinOperatorFactory = new HashSemiJoinOperatorFactory(2, new PlanNodeId("test"), setBuilderOperatorFactory.getSetProvider(), rowPagesBuilderProbe.getTypes(), 0);
    // expected
    MaterializedResult expected = resultBuilder(driverContext.getSession(), concat(probeTypes, ImmutableList.of(BOOLEAN))).row(30L, 0L, true).row(31L, 1L, false).row(32L, 2L, false).row(33L, 3L, false).row(34L, 4L, false).row(35L, 5L, true).row(36L, 6L, true).row(37L, 7L, true).row(38L, 8L, false).row(39L, 9L, false).build();
    OperatorAssertion.assertOperatorEquals(joinOperatorFactory, driverContext, probeInput, expected, hashEnabled, ImmutableList.of(probeTypes.size()));
}

Example 5 with JoinCompiler

use of com.facebook.presto.sql.gen.JoinCompiler in project presto by prestodb.

the class TestTopNRowNumberOperator method setUp.

@BeforeMethod
public void setUp() {
    executor = newCachedThreadPool(daemonThreadsNamed("test-%s"));
    driverContext = createTaskContext(executor, TEST_SESSION).addPipelineContext(0, true, true).addDriverContext();
    joinCompiler = new JoinCompiler();
}