Search in sources :

Example 16 with GroupProjection

use of io.crate.execution.dsl.projection.GroupProjection in project crate by crate.

the class ProjectorsTest method testProjectionsWithCorrectGranularityAreApplied.

@Test
public void testProjectionsWithCorrectGranularityAreApplied() {
    GroupProjection groupProjection = new GroupProjection(new ArrayList<>(), new ArrayList<>(), AggregateMode.ITER_FINAL, RowGranularity.SHARD);
    FilterProjection filterProjection = new FilterProjection(new InputColumn(0), Collections.emptyList());
    filterProjection.requiredGranularity(RowGranularity.DOC);
    Projectors projectors = new Projectors(Arrays.asList(filterProjection, groupProjection), UUID.randomUUID(), CoordinatorTxnCtx.systemTransactionContext(), RamAccounting.NO_ACCOUNTING, memoryManager, projectorFactory);
    assertThat(projectors.providesIndependentScroll(), is(true));
    assertThat(projectors.projectors.size(), is(1));
    assertThat(projectors.projectors.get(0), instanceOf(GroupingProjector.class));
}
Also used : FilterProjection(io.crate.execution.dsl.projection.FilterProjection) InputColumn(io.crate.expression.symbol.InputColumn) GroupingProjector(io.crate.execution.engine.aggregation.GroupingProjector) GroupProjection(io.crate.execution.dsl.projection.GroupProjection) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest) Test(org.junit.Test)

Example 17 with GroupProjection

use of io.crate.execution.dsl.projection.GroupProjection in project crate by crate.

the class DocValuesGroupByOptimizedIterator method tryOptimize.

@Nullable
static BatchIterator<Row> tryOptimize(Functions functions, IndexShard indexShard, DocTableInfo table, LuceneQueryBuilder luceneQueryBuilder, FieldTypeLookup fieldTypeLookup, DocInputFactory docInputFactory, RoutedCollectPhase collectPhase, CollectTask collectTask) {
    if (Symbols.containsColumn(collectPhase.toCollect(), DocSysColumns.SCORE) || Symbols.containsColumn(collectPhase.where(), DocSysColumns.SCORE)) {
        return null;
    }
    Collection<? extends Projection> shardProjections = shardProjections(collectPhase.projections());
    GroupProjection groupProjection = getSinglePartialGroupProjection(shardProjections);
    if (groupProjection == null) {
        return null;
    }
    ArrayList<Reference> columnKeyRefs = new ArrayList<>(groupProjection.keys().size());
    for (var key : groupProjection.keys()) {
        var docKeyRef = getKeyRef(collectPhase.toCollect(), key);
        if (docKeyRef == null) {
            // group by on non-reference
            return null;
        }
        var columnKeyRef = (Reference) DocReferences.inverseSourceLookup(docKeyRef);
        var keyFieldType = fieldTypeLookup.get(columnKeyRef.column().fqn());
        if (keyFieldType == null || !keyFieldType.hasDocValues()) {
            return null;
        } else {
            columnKeyRefs.add(columnKeyRef);
        }
    }
    // noinspection rawtypes
    List<DocValueAggregator> aggregators = DocValuesAggregates.createAggregators(functions, groupProjection.values(), collectPhase.toCollect(), collectTask.txnCtx().sessionSettings().searchPath(), table);
    if (aggregators == null) {
        return null;
    }
    ShardId shardId = indexShard.shardId();
    SharedShardContext sharedShardContext = collectTask.sharedShardContexts().getOrCreateContext(shardId);
    var searcher = sharedShardContext.acquireSearcher("group-by-doc-value-aggregates: " + formatSource(collectPhase));
    collectTask.addSearcher(sharedShardContext.readerId(), searcher);
    QueryShardContext queryShardContext = sharedShardContext.indexService().newQueryShardContext();
    InputFactory.Context<? extends LuceneCollectorExpression<?>> docCtx = docInputFactory.getCtx(collectTask.txnCtx());
    List<LuceneCollectorExpression<?>> keyExpressions = new ArrayList<>();
    for (var keyRef : columnKeyRefs) {
        keyExpressions.add((LuceneCollectorExpression<?>) docCtx.add(keyRef));
    }
    LuceneQueryBuilder.Context queryContext = luceneQueryBuilder.convert(collectPhase.where(), collectTask.txnCtx(), indexShard.mapperService(), indexShard.shardId().getIndexName(), queryShardContext, table, sharedShardContext.indexService().cache());
    if (columnKeyRefs.size() == 1) {
        return GroupByIterator.forSingleKey(aggregators, searcher.item(), columnKeyRefs.get(0), keyExpressions, collectTask.getRamAccounting(), collectTask.memoryManager(), collectTask.minNodeVersion(), queryContext.query(), new CollectorContext(sharedShardContext.readerId()));
    } else {
        return GroupByIterator.forManyKeys(aggregators, searcher.item(), columnKeyRefs, keyExpressions, collectTask.getRamAccounting(), collectTask.memoryManager(), collectTask.minNodeVersion(), queryContext.query(), new CollectorContext(sharedShardContext.readerId()));
    }
}
Also used : InputFactory(io.crate.expression.InputFactory) DocValueAggregator(io.crate.execution.engine.aggregation.DocValueAggregator) AtomicReference(java.util.concurrent.atomic.AtomicReference) Reference(io.crate.metadata.Reference) ArrayList(java.util.ArrayList) ShardId(org.elasticsearch.index.shard.ShardId) LuceneQueryBuilder(io.crate.lucene.LuceneQueryBuilder) QueryShardContext(org.elasticsearch.index.query.QueryShardContext) CollectorContext(io.crate.expression.reference.doc.lucene.CollectorContext) GroupProjection(io.crate.execution.dsl.projection.GroupProjection) SharedShardContext(io.crate.execution.jobs.SharedShardContext) LuceneCollectorExpression(io.crate.expression.reference.doc.lucene.LuceneCollectorExpression) Nullable(javax.annotation.Nullable)

Example 18 with GroupProjection

use of io.crate.execution.dsl.projection.GroupProjection in project crate by crate.

the class GroupByScalarPlannerTest method testGroupByWithScalarPlan.

@Test
public void testGroupByWithScalarPlan() throws Exception {
    Merge merge = e.plan("select id + 1 from users group by id");
    Collect collect = (Collect) merge.subPlan();
    RoutedCollectPhase collectPhase = ((RoutedCollectPhase) collect.collectPhase());
    assertEquals(DataTypes.LONG, collectPhase.outputTypes().get(0));
    assertThat(collectPhase.maxRowGranularity(), is(RowGranularity.DOC));
    assertThat(collectPhase.projections().size(), is(2));
    assertThat(collectPhase.projections().get(0), instanceOf(GroupProjection.class));
    assertThat(collectPhase.projections().get(0).requiredGranularity(), is(RowGranularity.SHARD));
    assertThat(collectPhase.projections().get(1), instanceOf(EvalProjection.class));
    assertThat(collectPhase.projections().get(1).outputs().get(0), instanceOf(Function.class));
    assertThat(collectPhase.toCollect(), contains(isReference("id", DataTypes.LONG)));
    GroupProjection groupProjection = (GroupProjection) collectPhase.projections().get(0);
    assertThat(groupProjection.keys().get(0).valueType(), is(DataTypes.LONG));
    assertThat(collectPhase.projections().get(1).outputs(), contains(isFunction("add")));
    MergePhase mergePhase = merge.mergePhase();
    assertEquals(DataTypes.LONG, mergePhase.inputTypes().iterator().next());
    assertEquals(DataTypes.LONG, mergePhase.outputTypes().get(0));
}
Also used : SymbolMatchers.isFunction(io.crate.testing.SymbolMatchers.isFunction) Function(io.crate.expression.symbol.Function) MergePhase(io.crate.execution.dsl.phases.MergePhase) Collect(io.crate.planner.node.dql.Collect) EvalProjection(io.crate.execution.dsl.projection.EvalProjection) GroupProjection(io.crate.execution.dsl.projection.GroupProjection) RoutedCollectPhase(io.crate.execution.dsl.phases.RoutedCollectPhase) Test(org.junit.Test) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest) RandomizedTest(com.carrotsearch.randomizedtesting.RandomizedTest)

Aggregations

GroupProjection (io.crate.execution.dsl.projection.GroupProjection)18 Test (org.junit.Test)14 CrateDummyClusterServiceUnitTest (io.crate.test.integration.CrateDummyClusterServiceUnitTest)13 RandomizedTest (com.carrotsearch.randomizedtesting.RandomizedTest)9 RoutedCollectPhase (io.crate.execution.dsl.phases.RoutedCollectPhase)9 Collect (io.crate.planner.node.dql.Collect)9 MergePhase (io.crate.execution.dsl.phases.MergePhase)7 EvalProjection (io.crate.execution.dsl.projection.EvalProjection)7 InputColumn (io.crate.expression.symbol.InputColumn)5 FilterProjection (io.crate.execution.dsl.projection.FilterProjection)4 Projection (io.crate.execution.dsl.projection.Projection)4 Symbol (io.crate.expression.symbol.Symbol)4 Merge (io.crate.planner.Merge)4 CollectPhase (io.crate.execution.dsl.phases.CollectPhase)3 OrderedTopNProjection (io.crate.execution.dsl.projection.OrderedTopNProjection)3 TopNProjection (io.crate.execution.dsl.projection.TopNProjection)3 CountAggregation (io.crate.execution.engine.aggregation.impl.CountAggregation)3 Aggregation (io.crate.expression.symbol.Aggregation)3 Reference (io.crate.metadata.Reference)3 TestingRowConsumer (io.crate.testing.TestingRowConsumer)3