Examples with PlanNodeId com.facebook.presto.spi.plan.PlanNodeId used on opensource projects

Example 86 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestDetermineSemiJoinDistributionType method testReplicatesWhenNotRestricted.

@Test
public void testReplicatesWhenNotRestricted() {
    int aRows = 10_000;
    int bRows = 10;
    PlanNodeStatsEstimate probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "A1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 10))).build();
    PlanNodeStatsEstimate buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "B1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000, 10))).build();
    // B table is small enough to be replicated in AUTOMATIC_RESTRICTED mode
    assertDetermineSemiJoinDistributionType().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).on(p -> p.semiJoin(p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), p.variable("A1"), p.variable("B1"), p.variable("output"), Optional.empty(), Optional.empty(), Optional.empty())).matches(semiJoin("A1", "B1", "output", Optional.of(REPLICATED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
    probeSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(aRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "A1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
    buildSideStatsEstimate = PlanNodeStatsEstimate.builder().setOutputRowCount(bRows).addVariableStatistics(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), "B1", BIGINT), new VariableStatsEstimate(0, 100, 0, 640000d * 10000, 10))).build();
    // B table exceeds AUTOMATIC_RESTRICTED limit therefore it is partitioned
    assertDetermineSemiJoinDistributionType().setSystemProperty(JOIN_DISTRIBUTION_TYPE, JoinDistributionType.AUTOMATIC.name()).setSystemProperty(JOIN_MAX_BROADCAST_TABLE_SIZE, "100MB").overrideStats("valuesA", probeSideStatsEstimate).overrideStats("valuesB", buildSideStatsEstimate).on(p -> p.semiJoin(p.values(new PlanNodeId("valuesA"), aRows, p.variable("A1", BIGINT)), p.values(new PlanNodeId("valuesB"), bRows, p.variable("B1", BIGINT)), p.variable("A1"), p.variable("B1"), p.variable("output"), Optional.empty(), Optional.empty(), Optional.empty())).matches(semiJoin("A1", "B1", "output", Optional.of(PARTITIONED), values(ImmutableMap.of("A1", 0)), values(ImmutableMap.of("B1", 0))));
}

Example 87 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class StatsAndCosts method getForSubplan.

public StatsAndCosts getForSubplan(PlanNode root) {
    Iterable<PlanNode> planIterator = Traverser.forTree(PlanNode::getSources).depthFirstPreOrder(root);
    ImmutableMap.Builder<PlanNodeId, PlanNodeStatsEstimate> filteredStats = ImmutableMap.builder();
    ImmutableMap.Builder<PlanNodeId, PlanCostEstimate> filteredCosts = ImmutableMap.builder();
    for (PlanNode node : planIterator) {
        if (stats.containsKey(node.getId())) {
            filteredStats.put(node.getId(), stats.get(node.getId()));
        }
        if (costs.containsKey(node.getId())) {
            filteredCosts.put(node.getId(), costs.get(node.getId()));
        }
    }
    return new StatsAndCosts(filteredStats.build(), filteredCosts.build());
}

Example 88 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class LocalQueryRunner method createDrivers.

private List<Driver> createDrivers(Session session, Plan plan, OutputFactory outputFactory, TaskContext taskContext) {
    if (printPlan) {
        System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata.getFunctionAndTypeManager(), plan.getStatsAndCosts(), session, 0, false));
    }
    SubPlan subplan = createSubPlans(session, plan, true);
    if (!subplan.getChildren().isEmpty()) {
        throw new AssertionError("Expected subplan to have no children");
    }
    LocalExecutionPlanner executionPlanner = new LocalExecutionPlanner(metadata, Optional.empty(), pageSourceManager, indexManager, partitioningProviderManager, nodePartitioningManager, pageSinkManager, distributedMetadataManager, expressionCompiler, pageFunctionCompiler, joinFilterFunctionCompiler, new IndexJoinLookupStats(), new TaskManagerConfig().setTaskConcurrency(4), new MemoryManagerConfig(), spillerFactory, singleStreamSpillerFactory, partitioningSpillerFactory, blockEncodingManager, new PagesIndex.TestingFactory(false), joinCompiler, new LookupJoinOperators(), new OrderingCompiler(), jsonCodec(TableCommitContext.class), new RowExpressionDeterminismEvaluator(metadata), new NoOpFragmentResultCacheManager(), new ObjectMapper(), standaloneSpillerFactory);
    // plan query
    StageExecutionDescriptor stageExecutionDescriptor = subplan.getFragment().getStageExecutionDescriptor();
    StreamingPlanSection streamingPlanSection = extractStreamingSections(subplan);
    checkState(streamingPlanSection.getChildren().isEmpty(), "expected no materialized exchanges");
    StreamingSubPlan streamingSubPlan = streamingPlanSection.getPlan();
    LocalExecutionPlan localExecutionPlan = executionPlanner.plan(taskContext, stageExecutionDescriptor, subplan.getFragment().getRoot(), subplan.getFragment().getPartitioningScheme(), subplan.getFragment().getTableScanSchedulingOrder(), outputFactory, Optional.empty(), new UnsupportedRemoteSourceFactory(), createTableWriteInfo(streamingSubPlan, metadata, session), false);
    // generate sources
    List<TaskSource> sources = new ArrayList<>();
    long sequenceId = 0;
    for (TableScanNode tableScan : findTableScanNodes(subplan.getFragment().getRoot())) {
        SplitSource splitSource = splitManager.getSplits(session, tableScan.getTable(), getSplitSchedulingStrategy(stageExecutionDescriptor, tableScan.getId()), WarningCollector.NOOP);
        ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
        while (!splitSource.isFinished()) {
            for (Split split : getNextBatch(splitSource)) {
                scheduledSplits.add(new ScheduledSplit(sequenceId++, tableScan.getId(), split));
            }
        }
        sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
    }
    // create drivers
    List<Driver> drivers = new ArrayList<>();
    Map<PlanNodeId, DriverFactory> driverFactoriesBySource = new HashMap<>();
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
        for (int i = 0; i < driverFactory.getDriverInstances().orElse(1); i++) {
            if (driverFactory.getSourceId().isPresent()) {
                checkState(driverFactoriesBySource.put(driverFactory.getSourceId().get(), driverFactory) == null);
            } else {
                DriverContext driverContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver(), false).addDriverContext();
                Driver driver = driverFactory.createDriver(driverContext);
                drivers.add(driver);
            }
        }
    }
    // add sources to the drivers
    Set<PlanNodeId> tableScanPlanNodeIds = ImmutableSet.copyOf(subplan.getFragment().getTableScanSchedulingOrder());
    for (TaskSource source : sources) {
        DriverFactory driverFactory = driverFactoriesBySource.get(source.getPlanNodeId());
        checkState(driverFactory != null);
        boolean partitioned = tableScanPlanNodeIds.contains(driverFactory.getSourceId().get());
        for (ScheduledSplit split : source.getSplits()) {
            DriverContext driverContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver(), partitioned).addDriverContext();
            Driver driver = driverFactory.createDriver(driverContext);
            driver.updateSource(new TaskSource(split.getPlanNodeId(), ImmutableSet.of(split), true));
            drivers.add(driver);
        }
    }
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
        driverFactory.noMoreDrivers();
    }
    return ImmutableList.copyOf(drivers);
}

Example 89 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestTableWriterOperator method testTableWriterInfo.

@Test
public void testTableWriterInfo() {
    PageSinkManager pageSinkManager = new PageSinkManager();
    pageSinkManager.addConnectorPageSinkProvider(CONNECTOR_ID, new ConstantPageSinkProvider(new TableWriteInfoTestPageSink()));
    TableWriterOperator tableWriterOperator = (TableWriterOperator) createTableWriterOperator(pageSinkManager, new DevNullOperatorFactory(1, new PlanNodeId("test")), ImmutableList.of(BIGINT, VARBINARY));
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(BIGINT);
    for (int i = 0; i < 100; i++) {
        rowPagesBuilder.addSequencePage(100, 0);
    }
    List<Page> pages = rowPagesBuilder.build();
    long peakMemoryUsage = 0;
    long validationCpuNanos = 0;
    for (int i = 0; i < pages.size(); i++) {
        Page page = pages.get(i);
        peakMemoryUsage += page.getRetainedSizeInBytes();
        validationCpuNanos += page.getPositionCount();
        tableWriterOperator.addInput(page);
        TableWriterInfo info = tableWriterOperator.getInfo();
        assertEquals(info.getPageSinkPeakMemoryUsage(), peakMemoryUsage);
        assertEquals((long) (info.getValidationCpuTime().getValue(NANOSECONDS)), validationCpuNanos);
    }
}

Example 90 with PlanNodeId

use of com.facebook.presto.spi.plan.PlanNodeId in project presto by prestodb.

the class TestTableWriterOperator method createTableWriterOperator.

private Operator createTableWriterOperator(PageSinkManager pageSinkManager, OperatorFactory statisticsAggregation, List<Type> outputTypes, Session session, TaskMetadataContext taskMetadataContext, DriverContext driverContext) {
    List<String> notNullColumnNames = new ArrayList<>(1);
    notNullColumnNames.add(null);
    TableWriterOperatorFactory factory = new TableWriterOperatorFactory(0, new PlanNodeId("test"), pageSinkManager, new ConnectorMetadataUpdaterManager(), taskMetadataContext, new CreateHandle(new OutputTableHandle(CONNECTOR_ID, new ConnectorTransactionHandle() {
    }, new ConnectorOutputTableHandle() {
    }), new SchemaTableName("testSchema", "testTable")), ImmutableList.of(0), notNullColumnNames, session, statisticsAggregation, outputTypes, TABLE_COMMIT_CONTEXT_CODEC, NO_COMMIT);
    return factory.createOperator(driverContext);
}