Examples with GroupProjection io.crate.execution.dsl.projection.GroupProjection used on opensource projects

Example 11 with GroupProjection

use of io.crate.execution.dsl.projection.GroupProjection in project crate by crate.

the class GroupByPlannerTest method testGroupByOnNodeLevel.

@Test
public void testGroupByOnNodeLevel() throws Exception {
    var e = SQLExecutor.builder(clusterService, 2, RandomizedTest.getRandom(), List.of()).build();
    Collect collect = e.plan("select count(*), name from sys.nodes group by name");
    assertThat("number of nodeIds must be 1, otherwise there must be a merge", collect.resultDescription().nodeIds().size(), is(1));
    RoutedCollectPhase collectPhase = ((RoutedCollectPhase) collect.collectPhase());
    assertThat(collectPhase.projections(), contains(instanceOf(GroupProjection.class), instanceOf(EvalProjection.class)));
    GroupProjection groupProjection = (GroupProjection) collectPhase.projections().get(0);
    assertThat(Symbols.typeView(groupProjection.outputs()), contains(is(DataTypes.STRING), is(DataTypes.LONG)));
    assertThat(Symbols.typeView(collectPhase.projections().get(1).outputs()), contains(is(DataTypes.LONG), is(DataTypes.STRING)));
}

Example 12 with GroupProjection

use of io.crate.execution.dsl.projection.GroupProjection in project crate by crate.

the class MergeNodeTest method testSerialization.

@Test
public void testSerialization() throws Exception {
    List<Symbol> keys = Collections.singletonList(new InputColumn(0, DataTypes.STRING));
    List<Aggregation> aggregations = Collections.singletonList(new Aggregation(CountAggregation.COUNT_STAR_SIGNATURE, CountAggregation.COUNT_STAR_SIGNATURE.getReturnType().createType(), Collections.emptyList()));
    GroupProjection groupProjection = new GroupProjection(keys, aggregations, AggregateMode.PARTIAL_FINAL, RowGranularity.CLUSTER);
    TopNProjection topNProjection = new TopNProjection(10, 0, Symbols.typeView(groupProjection.outputs()));
    List<Projection> projections = Arrays.asList(groupProjection, topNProjection);
    MergePhase node = new MergePhase(UUID.randomUUID(), 0, "merge", 2, 1, Set.of("node1", "node2"), List.of(DataTypes.UNDEFINED, DataTypes.STRING), projections, DistributionInfo.DEFAULT_BROADCAST, null);
    BytesStreamOutput output = new BytesStreamOutput();
    node.writeTo(output);
    StreamInput input = output.bytes().streamInput();
    MergePhase node2 = new MergePhase(input);
    assertThat(node.numUpstreams(), is(node2.numUpstreams()));
    assertThat(node.nodeIds(), is(node2.nodeIds()));
    assertThat(node.jobId(), is(node2.jobId()));
    assertEquals(node.inputTypes(), node2.inputTypes());
    assertThat(node.phaseId(), is(node2.phaseId()));
    assertThat(node.distributionInfo(), is(node2.distributionInfo()));
}

Example 13 with GroupProjection

use of io.crate.execution.dsl.projection.GroupProjection in project crate by crate.

the class GroupByOptimizedIterator method tryOptimizeSingleStringKey.

@Nullable
static BatchIterator<Row> tryOptimizeSingleStringKey(IndexShard indexShard, DocTableInfo table, LuceneQueryBuilder luceneQueryBuilder, FieldTypeLookup fieldTypeLookup, BigArrays bigArrays, InputFactory inputFactory, DocInputFactory docInputFactory, RoutedCollectPhase collectPhase, CollectTask collectTask) {
    Collection<? extends Projection> shardProjections = shardProjections(collectPhase.projections());
    GroupProjection groupProjection = getSingleStringKeyGroupProjection(shardProjections);
    if (groupProjection == null) {
        return null;
    }
    assert groupProjection.keys().size() == 1 : "Must have 1 key if getSingleStringKeyGroupProjection returned a projection";
    Reference keyRef = getKeyRef(collectPhase.toCollect(), groupProjection.keys().get(0));
    if (keyRef == null) {
        // group by on non-reference
        return null;
    }
    keyRef = (Reference) DocReferences.inverseSourceLookup(keyRef);
    MappedFieldType keyFieldType = fieldTypeLookup.get(keyRef.column().fqn());
    if (keyFieldType == null || !keyFieldType.hasDocValues()) {
        return null;
    }
    if (Symbols.containsColumn(collectPhase.toCollect(), DocSysColumns.SCORE) || Symbols.containsColumn(collectPhase.where(), DocSysColumns.SCORE)) {
        // to keep the optimized implementation a bit simpler
        return null;
    }
    if (hasHighCardinalityRatio(() -> indexShard.acquireSearcher("group-by-cardinality-check"), keyFieldType.name())) {
        return null;
    }
    ShardId shardId = indexShard.shardId();
    SharedShardContext sharedShardContext = collectTask.sharedShardContexts().getOrCreateContext(shardId);
    var searcher = sharedShardContext.acquireSearcher("group-by-ordinals:" + formatSource(collectPhase));
    collectTask.addSearcher(sharedShardContext.readerId(), searcher);
    final QueryShardContext queryShardContext = sharedShardContext.indexService().newQueryShardContext();
    InputFactory.Context<? extends LuceneCollectorExpression<?>> docCtx = docInputFactory.getCtx(collectTask.txnCtx());
    docCtx.add(collectPhase.toCollect().stream()::iterator);
    InputFactory.Context<CollectExpression<Row, ?>> ctxForAggregations = inputFactory.ctxForAggregations(collectTask.txnCtx());
    ctxForAggregations.add(groupProjection.values());
    final List<CollectExpression<Row, ?>> aggExpressions = ctxForAggregations.expressions();
    List<AggregationContext> aggregations = ctxForAggregations.aggregations();
    List<? extends LuceneCollectorExpression<?>> expressions = docCtx.expressions();
    RamAccounting ramAccounting = collectTask.getRamAccounting();
    CollectorContext collectorContext = new CollectorContext(sharedShardContext.readerId());
    InputRow inputRow = new InputRow(docCtx.topLevelInputs());
    LuceneQueryBuilder.Context queryContext = luceneQueryBuilder.convert(collectPhase.where(), collectTask.txnCtx(), indexShard.mapperService(), indexShard.shardId().getIndexName(), queryShardContext, table, sharedShardContext.indexService().cache());
    return getIterator(bigArrays, searcher.item(), keyRef.column().fqn(), aggregations, expressions, aggExpressions, ramAccounting, collectTask.memoryManager(), collectTask.minNodeVersion(), inputRow, queryContext.query(), collectorContext, groupProjection.mode());
}

Example 14 with GroupProjection

use of io.crate.execution.dsl.projection.GroupProjection in project crate by crate.

the class ProjectionToProjectorVisitorTest method testGroupProjector.

@Test
public void testGroupProjector() throws Exception {
    // in(0)  in(1)      in(0),      in(2)
    // select  race, avg(age), count(race), gender  ... group by race, gender
    List<Symbol> keys = Arrays.asList(new InputColumn(0, DataTypes.STRING), new InputColumn(2, DataTypes.STRING));
    List<Aggregation> aggregations = Arrays.asList(new Aggregation(avgSignature, avgSignature.getReturnType().createType(), Collections.singletonList(new InputColumn(1))), new Aggregation(CountAggregation.SIGNATURE, CountAggregation.SIGNATURE.getReturnType().createType(), Collections.singletonList(new InputColumn(0))));
    GroupProjection projection = new GroupProjection(keys, aggregations, AggregateMode.ITER_FINAL, RowGranularity.CLUSTER);
    Projector projector = visitor.create(projection, txnCtx, RamAccounting.NO_ACCOUNTING, memoryManager, UUID.randomUUID());
    assertThat(projector, instanceOf(GroupingProjector.class));
    // use a topN projection in order to get sorted outputs
    List<Symbol> outputs = Arrays.asList(new InputColumn(0, DataTypes.STRING), new InputColumn(1, DataTypes.STRING), new InputColumn(2, DataTypes.DOUBLE), new InputColumn(3, DataTypes.LONG));
    OrderedTopNProjection topNProjection = new OrderedTopNProjection(10, 0, outputs, List.of(new InputColumn(2, DataTypes.DOUBLE)), new boolean[] { false }, new boolean[] { false });
    Projector topNProjector = visitor.create(topNProjection, txnCtx, RamAccounting.NO_ACCOUNTING, memoryManager, UUID.randomUUID());
    String human = "human";
    String vogon = "vogon";
    String male = "male";
    String female = "female";
    List<Object[]> rows = new ArrayList<>();
    rows.add($(human, 34, male));
    rows.add($(human, 22, female));
    rows.add($(vogon, 40, male));
    rows.add($(vogon, 48, male));
    rows.add($(human, 34, male));
    BatchIterator<Row> batchIterator = topNProjector.apply(projector.apply(InMemoryBatchIterator.of(new CollectionBucket(rows), SENTINEL, true)));
    TestingRowConsumer consumer = new TestingRowConsumer();
    consumer.accept(batchIterator, null);
    Bucket bucket = consumer.getBucket();
    assertThat(bucket, contains(isRow(human, female, 22.0, 1L), isRow(human, male, 34.0, 2L), isRow(vogon, male, 44.0, 2L)));
}

Example 15 with GroupProjection

use of io.crate.execution.dsl.projection.GroupProjection in project crate by crate.

the class ProjectingRowConsumerTest method testConsumerRequiresScrollAndProjectorsSupportScrolling.

@Test
public void testConsumerRequiresScrollAndProjectorsSupportScrolling() {
    GroupProjection groupProjection = new GroupProjection(new ArrayList<>(), new ArrayList<>(), AggregateMode.ITER_FINAL, RowGranularity.SHARD);
    RowConsumer delegateConsumerRequiresScroll = new DummyRowConsumer(true);
    RowConsumer projectingConsumer = ProjectingRowConsumer.create(delegateConsumerRequiresScroll, Collections.singletonList(groupProjection), UUID.randomUUID(), txnCtx, RamAccounting.NO_ACCOUNTING, memoryManager, projectorFactory);
    assertThat(projectingConsumer.requiresScroll(), is(false));
}