Examples with RoutedCollectPhase io.crate.planner.node.dql.RoutedCollectPhase used on opensource projects

Example 26 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class ShardCollectSource method getCollector.

@Override
public CrateCollector getCollector(CollectPhase phase, BatchConsumer lastConsumer, JobCollectContext jobCollectContext) {
    RoutedCollectPhase collectPhase = (RoutedCollectPhase) phase;
    RoutedCollectPhase normalizedPhase = collectPhase.normalize(nodeNormalizer, null);
    String localNodeId = clusterService.localNode().getId();
    BatchConsumer firstConsumer = ProjectingBatchConsumer.create(lastConsumer, Projections.nodeProjections(normalizedPhase.projections()), collectPhase.jobId(), jobCollectContext.queryPhaseRamAccountingContext(), sharedProjectorFactory);
    if (normalizedPhase.maxRowGranularity() == RowGranularity.SHARD) {
        // The getShardsCollector method always only uses a single RowReceiver and not one per shard)
        return getShardsCollector(collectPhase, normalizedPhase, localNodeId, firstConsumer);
    }
    OrderBy orderBy = normalizedPhase.orderBy();
    if (normalizedPhase.maxRowGranularity() == RowGranularity.DOC && orderBy != null) {
        return createMultiShardScoreDocCollector(normalizedPhase, firstConsumer, jobCollectContext, localNodeId);
    }
    // actual shards might be less if table is partitioned and a partition has been deleted meanwhile
    final int maxNumShards = normalizedPhase.routing().numShards(localNodeId);
    boolean hasShardProjections = Projections.hasAnyShardProjections(normalizedPhase.projections());
    Map<String, Map<String, List<Integer>>> locations = normalizedPhase.routing().locations();
    final List<CrateCollector.Builder> builders = new ArrayList<>(maxNumShards);
    Map<String, List<Integer>> indexShards = locations.get(localNodeId);
    if (indexShards != null) {
        builders.addAll(getDocCollectors(jobCollectContext, normalizedPhase, lastConsumer.requiresScroll(), indexShards));
    }
    switch(builders.size()) {
        case 0:
            return RowsCollector.empty(firstConsumer);
        case 1:
            CrateCollector.Builder collectorBuilder = builders.iterator().next();
            return collectorBuilder.build(collectorBuilder.applyProjections(firstConsumer));
        default:
            if (hasShardProjections) {
                // in order to process shard-based projections concurrently
                return new CompositeCollector(builders, firstConsumer, iterators -> new AsyncCompositeBatchIterator(executor, iterators));
            } else {
                return new CompositeCollector(builders, firstConsumer, CompositeBatchIterator::new);
            }
    }
}

Example 27 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class SingleRowSource method getCollector.

@Override
public CrateCollector getCollector(CollectPhase phase, BatchConsumer consumer, JobCollectContext jobCollectContext) {
    RoutedCollectPhase collectPhase = (RoutedCollectPhase) phase;
    collectPhase = collectPhase.normalize(clusterNormalizer, null);
    if (collectPhase.whereClause().noMatch()) {
        return RowsCollector.empty(consumer);
    }
    assert !collectPhase.whereClause().hasQuery() : "WhereClause should have been normalized to either MATCH_ALL or NO_MATCH";
    InputFactory inputFactory = new InputFactory(functions);
    InputFactory.Context<CollectExpression<Row, ?>> ctx = inputFactory.ctxForInputColumns(collectPhase.toCollect());
    return RowsCollector.single(new InputRow(ctx.topLevelInputs()), consumer);
}

Example 28 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class ShardCollectorProvider method getCollectorBuilder.

/**
     * Create a CrateCollector.Builder to collect rows from a shard.
     * <p>
     * This also creates all shard-level projectors.
     * The RowReceiver that is used for {@link CrateCollector.Builder#build(BatchConsumer)}
     * should be the first node-level projector.
     */
public CrateCollector.Builder getCollectorBuilder(RoutedCollectPhase collectPhase, boolean requiresScroll, JobCollectContext jobCollectContext) throws Exception {
    assert collectPhase.orderBy() == null : "getDocCollector shouldn't be called if there is an orderBy on the collectPhase";
    RoutedCollectPhase normalizedCollectNode = collectPhase.normalize(shardNormalizer, null);
    final CrateCollector.Builder builder;
    if (normalizedCollectNode.whereClause().noMatch()) {
        builder = RowsCollector.emptyBuilder();
    } else {
        assert normalizedCollectNode.maxRowGranularity() == RowGranularity.DOC : "granularity must be DOC";
        builder = getBuilder(normalizedCollectNode, requiresScroll, jobCollectContext);
    }
    Collection<? extends Projection> shardProjections = Projections.shardProjections(collectPhase.projections());
    if (shardProjections.isEmpty()) {
        return builder;
    } else {
        return new CrateCollector.Builder() {

            @Override
            public CrateCollector build(BatchConsumer batchConsumer) {
                return builder.build(batchConsumer);
            }

            @Override
            public BatchConsumer applyProjections(BatchConsumer consumer) {
                return ProjectingBatchConsumer.create(consumer, shardProjections, normalizedCollectNode.jobId(), jobCollectContext.queryPhaseRamAccountingContext(), projectorFactory);
            }
        };
    }
}

Example 29 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class DistributingDownstreamFactoryTest method createDownstream.

private BatchConsumer createDownstream(Set<String> downstreamExecutionNodes) {
    UUID jobId = UUID.randomUUID();
    Routing routing = new Routing(TreeMapBuilder.<String, Map<String, List<Integer>>>newMapBuilder().put("n1", TreeMapBuilder.<String, List<Integer>>newMapBuilder().put("i1", Arrays.asList(1, 2)).map()).map());
    RoutedCollectPhase collectPhase = new RoutedCollectPhase(jobId, 1, "collect", routing, RowGranularity.DOC, ImmutableList.<Symbol>of(), ImmutableList.<Projection>of(), WhereClause.MATCH_ALL, DistributionInfo.DEFAULT_MODULO);
    MergePhase mergePhase = new MergePhase(jobId, 2, "merge", 1, Collections.emptyList(), ImmutableList.<DataType>of(LongType.INSTANCE), ImmutableList.<Projection>of(), DistributionInfo.DEFAULT_BROADCAST, null);
    mergePhase.executionNodes(downstreamExecutionNodes);
    NodeOperation nodeOperation = NodeOperation.withDownstream(collectPhase, mergePhase, (byte) 0, "nodeName");
    return rowDownstreamFactory.create(nodeOperation, collectPhase.distributionInfo(), jobId, Paging.PAGE_SIZE);
}

Example 30 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class GlobalAggregateConsumer method globalAggregates.

/**
     * Create a Merge(Collect) plan.
     *
     * iter->partial aggregations on use {@code projectionGranularity} granularity
     */
private static Plan globalAggregates(Functions functions, QueriedTableRelation table, ConsumerContext context, RowGranularity projectionGranularity) {
    QuerySpec querySpec = table.querySpec();
    if (querySpec.groupBy().isPresent() || !querySpec.hasAggregates()) {
        return null;
    }
    // global aggregate: collect and partial aggregate on C and final agg on H
    Planner.Context plannerContext = context.plannerContext();
    validateAggregationOutputs(table.tableRelation(), querySpec.outputs());
    ProjectionBuilder projectionBuilder = new ProjectionBuilder(functions, querySpec);
    SplitPoints splitPoints = projectionBuilder.getSplitPoints();
    AggregationProjection ap = projectionBuilder.aggregationProjection(splitPoints.leaves(), splitPoints.aggregates(), Aggregation.Step.ITER, Aggregation.Step.PARTIAL, projectionGranularity);
    RoutedCollectPhase collectPhase = RoutedCollectPhase.forQueriedTable(plannerContext, table, splitPoints.leaves(), ImmutableList.of(ap));
    Collect collect = new Collect(collectPhase, TopN.NO_LIMIT, 0, ap.outputs().size(), 1, null);
    AggregationProjection aggregationProjection = projectionBuilder.aggregationProjection(splitPoints.aggregates(), splitPoints.aggregates(), Aggregation.Step.PARTIAL, Aggregation.Step.FINAL, RowGranularity.CLUSTER);
    List<Projection> postAggregationProjections = createPostAggregationProjections(querySpec, splitPoints, plannerContext);
    postAggregationProjections.add(0, aggregationProjection);
    return createMerge(collect, plannerContext, postAggregationProjections);
}