Search in sources :

Example 1 with EvalProjection

use of io.crate.execution.dsl.projection.EvalProjection in project crate by crate.

the class Get method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> hints, ProjectionBuilder projectionBuilder, int limitHint, int offsetHint, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    HashMap<String, Map<ShardId, List<PKAndVersion>>> idsByShardByNode = new HashMap<>();
    DocTableInfo docTableInfo = tableRelation.tableInfo();
    for (DocKeys.DocKey docKey : docKeys) {
        String id = docKey.getId(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        if (id == null) {
            continue;
        }
        List<String> partitionValues = docKey.getPartitionValues(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        String indexName = indexName(docTableInfo, partitionValues);
        String routing = docKey.getRouting(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults);
        ShardRouting shardRouting;
        try {
            shardRouting = plannerContext.resolveShard(indexName, id, routing);
        } catch (IndexNotFoundException e) {
            if (docTableInfo.isPartitioned()) {
                continue;
            }
            throw e;
        }
        String currentNodeId = shardRouting.currentNodeId();
        if (currentNodeId == null) {
            // If relocating is fast enough this will work, otherwise it will result in a shard failure which
            // will cause a statement retry
            currentNodeId = shardRouting.relocatingNodeId();
            if (currentNodeId == null) {
                throw new ShardNotFoundException(shardRouting.shardId());
            }
        }
        Map<ShardId, List<PKAndVersion>> idsByShard = idsByShardByNode.get(currentNodeId);
        if (idsByShard == null) {
            idsByShard = new HashMap<>();
            idsByShardByNode.put(currentNodeId, idsByShard);
        }
        List<PKAndVersion> pkAndVersions = idsByShard.get(shardRouting.shardId());
        if (pkAndVersions == null) {
            pkAndVersions = new ArrayList<>();
            idsByShard.put(shardRouting.shardId(), pkAndVersions);
        }
        long version = docKey.version(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(Versions.MATCH_ANY);
        long sequenceNumber = docKey.sequenceNo(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_SEQ_NO);
        long primaryTerm = docKey.primaryTerm(plannerContext.transactionContext(), plannerContext.nodeContext(), params, subQueryResults).orElse(SequenceNumbers.UNASSIGNED_PRIMARY_TERM);
        pkAndVersions.add(new PKAndVersion(id, version, sequenceNumber, primaryTerm));
    }
    var docKeyColumns = new ArrayList<>(docTableInfo.primaryKey());
    docKeyColumns.addAll(docTableInfo.partitionedBy());
    docKeyColumns.add(docTableInfo.clusteredBy());
    docKeyColumns.add(DocSysColumns.VERSION);
    docKeyColumns.add(DocSysColumns.SEQ_NO);
    docKeyColumns.add(DocSysColumns.PRIMARY_TERM);
    var binder = new SubQueryAndParamBinder(params, subQueryResults);
    List<Symbol> boundOutputs = Lists2.map(outputs, binder);
    var boundQuery = binder.apply(query);
    // Collect all columns which are used inside the query
    // If the query contains only DocKeys, no filter is needed as all DocKeys are handled by the PKLookupOperation
    AtomicBoolean requiresAdditionalFilteringOnNonDocKeyColumns = new AtomicBoolean(false);
    var toCollectSet = new LinkedHashSet<>(boundOutputs);
    Consumer<Reference> addRefIfMatch = ref -> {
        toCollectSet.add(ref);
        if (docKeyColumns.contains(ref.column()) == false) {
            requiresAdditionalFilteringOnNonDocKeyColumns.set(true);
        }
    };
    RefVisitor.visitRefs(boundQuery, addRefIfMatch);
    var toCollect = boundOutputs;
    ArrayList<Projection> projections = new ArrayList<>();
    if (requiresAdditionalFilteringOnNonDocKeyColumns.get()) {
        toCollect = List.copyOf(toCollectSet);
        var filterProjection = ProjectionBuilder.filterProjection(toCollect, boundQuery);
        filterProjection.requiredGranularity(RowGranularity.SHARD);
        projections.add(filterProjection);
        // reduce outputs which have been added for the filter projection
        var evalProjection = new EvalProjection(InputColumn.mapToInputColumns(boundOutputs), RowGranularity.SHARD);
        projections.add(evalProjection);
    }
    var collect = new Collect(new PKLookupPhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), docTableInfo.partitionedBy(), toCollect, idsByShardByNode), TopN.NO_LIMIT, 0, toCollect.size(), docKeys.size(), null);
    for (var projection : projections) {
        collect.addProjection(projection);
    }
    return collect;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) ShardRouting(org.elasticsearch.cluster.routing.ShardRouting) ShardId(org.elasticsearch.index.shard.ShardId) IndexParts(io.crate.metadata.IndexParts) InputColumn(io.crate.expression.symbol.InputColumn) Versions(org.elasticsearch.common.lucene.uid.Versions) RelationName(io.crate.metadata.RelationName) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) PartitionName(io.crate.metadata.PartitionName) ShardNotFoundException(org.elasticsearch.index.shard.ShardNotFoundException) ArrayList(java.util.ArrayList) RefVisitor(io.crate.expression.symbol.RefVisitor) TableStats(io.crate.statistics.TableStats) IndexNotFoundException(org.elasticsearch.index.IndexNotFoundException) Map(java.util.Map) SelectSymbol(io.crate.expression.symbol.SelectSymbol) PKLookupPhase(io.crate.execution.dsl.phases.PKLookupPhase) TopN(io.crate.execution.engine.pipeline.TopN) Nullable(javax.annotation.Nullable) LinkedHashSet(java.util.LinkedHashSet) ProjectionBuilder(io.crate.execution.dsl.projection.builder.ProjectionBuilder) DocSysColumns(io.crate.metadata.doc.DocSysColumns) DocTableInfo(io.crate.metadata.doc.DocTableInfo) DocKeys(io.crate.analyze.where.DocKeys) SequenceNumbers(org.elasticsearch.index.seqno.SequenceNumbers) Collection(java.util.Collection) Reference(io.crate.metadata.Reference) Set(java.util.Set) Lists2(io.crate.common.collections.Lists2) ExecutionPlan(io.crate.planner.ExecutionPlan) Consumer(java.util.function.Consumer) List(java.util.List) OrderBy(io.crate.analyze.OrderBy) RowGranularity(io.crate.metadata.RowGranularity) DocTableRelation(io.crate.analyze.relations.DocTableRelation) Row(io.crate.data.Row) Projection(io.crate.execution.dsl.projection.Projection) Symbol(io.crate.expression.symbol.Symbol) AbstractTableRelation(io.crate.analyze.relations.AbstractTableRelation) PlannerContext(io.crate.planner.PlannerContext) Collect(io.crate.planner.node.dql.Collect) EvalProjection(io.crate.execution.dsl.projection.EvalProjection) DocTableInfo(io.crate.metadata.doc.DocTableInfo) DocKeys(io.crate.analyze.where.DocKeys) HashMap(java.util.HashMap) Collect(io.crate.planner.node.dql.Collect) SelectSymbol(io.crate.expression.symbol.SelectSymbol) Symbol(io.crate.expression.symbol.Symbol) ArrayList(java.util.ArrayList) Projection(io.crate.execution.dsl.projection.Projection) EvalProjection(io.crate.execution.dsl.projection.EvalProjection) ShardId(org.elasticsearch.index.shard.ShardId) PKLookupPhase(io.crate.execution.dsl.phases.PKLookupPhase) ArrayList(java.util.ArrayList) List(java.util.List) Reference(io.crate.metadata.Reference) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ShardNotFoundException(org.elasticsearch.index.shard.ShardNotFoundException) EvalProjection(io.crate.execution.dsl.projection.EvalProjection) IndexNotFoundException(org.elasticsearch.index.IndexNotFoundException) ShardRouting(org.elasticsearch.cluster.routing.ShardRouting) HashMap(java.util.HashMap) Map(java.util.Map)

Example 2 with EvalProjection

use of io.crate.execution.dsl.projection.EvalProjection in project crate by crate.

the class TopNDistinct method build.

@Override
public ExecutionPlan build(PlannerContext plannerContext, Set<PlanHint> planHints, ProjectionBuilder projectionBuilder, int limitHint, int offsetHint, @Nullable OrderBy order, @Nullable Integer pageSizeHint, Row params, SubQueryResults subQueryResults) {
    var executionPlan = source.build(plannerContext, planHints, projectionBuilder, TopN.NO_LIMIT, TopN.NO_OFFSET, null, null, params, subQueryResults);
    if (executionPlan.resultDescription().hasRemainingLimitOrOffset()) {
        executionPlan = Merge.ensureOnHandler(executionPlan, plannerContext);
    }
    if (!source.outputs().equals(outputs)) {
        EvalProjection evalProjection = new EvalProjection(InputColumns.create(outputs, new InputColumns.SourceSymbols(source.outputs())));
        executionPlan.addProjection(evalProjection);
    }
    int limit = DataTypes.INTEGER.sanitizeValue(evaluate(plannerContext.transactionContext(), plannerContext.nodeContext(), this.limit, params, subQueryResults));
    int offset = DataTypes.INTEGER.sanitizeValue(evaluate(plannerContext.transactionContext(), plannerContext.nodeContext(), this.offset, params, subQueryResults));
    var inputColOutputs = InputColumn.mapToInputColumns(outputs);
    executionPlan.addProjection(new TopNDistinctProjection(limit + offset, inputColOutputs, source.preferShardProjections() ? RowGranularity.SHARD : RowGranularity.CLUSTER));
    boolean onHandler = ExecutionPhases.executesOnHandler(plannerContext.handlerNode(), executionPlan.resultDescription().nodeIds());
    if (!onHandler || source.preferShardProjections()) {
        if (!onHandler) {
            executionPlan = Merge.ensureOnHandler(executionPlan, plannerContext);
        }
        TopNDistinctProjection topNDistinct = new TopNDistinctProjection(limit + offset, inputColOutputs, RowGranularity.CLUSTER);
        executionPlan.addProjection(topNDistinct);
    }
    if (offset > 0) {
        // TopNDistinctProjection outputs a distinct result-set,
        // That allows us to use the TopNProjection to apply the offset
        executionPlan.addProjection(new TopNProjection(limit, offset, Symbols.typeView(inputColOutputs)));
    }
    return executionPlan;
}
Also used : EvalProjection(io.crate.execution.dsl.projection.EvalProjection) TopNProjection(io.crate.execution.dsl.projection.TopNProjection) TopNDistinctProjection(io.crate.execution.dsl.projection.TopNDistinctProjection)

Example 3 with EvalProjection

use of io.crate.execution.dsl.projection.EvalProjection in project crate by crate.

the class InsertFromSubQueryPlanner method plan.

public static LogicalPlan plan(AnalyzedInsertStatement statement, PlannerContext plannerContext, LogicalPlanner logicalPlanner, SubqueryPlanner subqueryPlanner) {
    if (statement.outputs() != null && !plannerContext.clusterState().getNodes().getMinNodeVersion().onOrAfter(Version.V_4_2_0)) {
        throw new UnsupportedFeatureException(RETURNING_VERSION_ERROR_MSG);
    }
    List<Reference> targetColsExclPartitionCols = new ArrayList<>(statement.columns().size() - statement.tableInfo().partitionedBy().size());
    for (Reference column : statement.columns()) {
        if (statement.tableInfo().partitionedBy().contains(column.column())) {
            continue;
        }
        targetColsExclPartitionCols.add(column);
    }
    List<Symbol> columnSymbols = InputColumns.create(targetColsExclPartitionCols, new InputColumns.SourceSymbols(statement.columns()));
    // if fields are null default to number of rows imported
    var outputs = statement.outputs() == null ? List.of(new InputColumn(0, DataTypes.LONG)) : statement.outputs();
    ColumnIndexWriterProjection indexWriterProjection = new ColumnIndexWriterProjection(statement.tableInfo().ident(), null, statement.tableInfo().primaryKey(), statement.columns(), targetColsExclPartitionCols, columnSymbols, statement.isIgnoreDuplicateKeys(), statement.onDuplicateKeyAssignments(), statement.primaryKeySymbols(), statement.partitionedBySymbols(), statement.tableInfo().clusteredBy(), statement.clusteredBySymbol(), Settings.EMPTY, statement.tableInfo().isPartitioned(), outputs, statement.outputs() == null ? List.of() : statement.outputs());
    LogicalPlan plannedSubQuery = logicalPlanner.plan(statement.subQueryRelation(), plannerContext, subqueryPlanner, true);
    EvalProjection castOutputs = EvalProjection.castValues(Symbols.typeView(statement.columns()), plannedSubQuery.outputs());
    return new Insert(plannedSubQuery, indexWriterProjection, castOutputs);
}
Also used : InputColumns(io.crate.execution.dsl.projection.builder.InputColumns) UnsupportedFeatureException(io.crate.exceptions.UnsupportedFeatureException) Reference(io.crate.metadata.Reference) Symbol(io.crate.expression.symbol.Symbol) ArrayList(java.util.ArrayList) ColumnIndexWriterProjection(io.crate.execution.dsl.projection.ColumnIndexWriterProjection) Insert(io.crate.planner.operators.Insert) EvalProjection(io.crate.execution.dsl.projection.EvalProjection) InputColumn(io.crate.expression.symbol.InputColumn) LogicalPlan(io.crate.planner.operators.LogicalPlan)

Example 4 with EvalProjection

use of io.crate.execution.dsl.projection.EvalProjection in project crate by crate.

the class GroupByPlannerTest method testGroupByWithAggregationPlan.

@Test
public void testGroupByWithAggregationPlan() throws Exception {
    var e = SQLExecutor.builder(clusterService, 2, RandomizedTest.getRandom(), List.of()).addTable(TableDefinitions.USER_TABLE_DEFINITION).build();
    Merge distributedGroupByMerge = e.plan("select count(*), name from users group by name");
    Merge reducerMerge = (Merge) distributedGroupByMerge.subPlan();
    // distributed collect
    RoutedCollectPhase collectPhase = ((RoutedCollectPhase) ((Collect) reducerMerge.subPlan()).collectPhase());
    assertThat(collectPhase.maxRowGranularity(), is(RowGranularity.DOC));
    assertThat(collectPhase.nodeIds().size(), is(2));
    assertThat(collectPhase.toCollect().size(), is(1));
    assertThat(collectPhase.projections().size(), is(1));
    assertThat(collectPhase.projections().get(0), instanceOf(GroupProjection.class));
    assertThat(collectPhase.outputTypes().size(), is(2));
    assertEquals(DataTypes.STRING, collectPhase.outputTypes().get(0));
    assertEquals(CountAggregation.LongStateType.INSTANCE, collectPhase.outputTypes().get(1));
    MergePhase mergePhase = reducerMerge.mergePhase();
    assertThat(mergePhase.numUpstreams(), is(2));
    assertThat(mergePhase.nodeIds().size(), is(2));
    assertEquals(mergePhase.inputTypes(), collectPhase.outputTypes());
    // for function evaluation and column-reordering there is always a EvalProjection
    assertThat(mergePhase.projections().size(), is(2));
    assertThat(mergePhase.projections().get(1), instanceOf(EvalProjection.class));
    assertThat(mergePhase.projections().get(0), instanceOf(GroupProjection.class));
    GroupProjection groupProjection = (GroupProjection) mergePhase.projections().get(0);
    InputColumn inputColumn = (InputColumn) groupProjection.values().get(0).inputs().get(0);
    assertThat(inputColumn.index(), is(1));
    assertThat(mergePhase.outputTypes().size(), is(2));
    assertEquals(DataTypes.LONG, mergePhase.outputTypes().get(0));
    assertEquals(DataTypes.STRING, mergePhase.outputTypes().get(1));
    MergePhase localMerge = distributedGroupByMerge.mergePhase();
    assertThat(localMerge.numUpstreams(), is(2));
    assertThat(localMerge.nodeIds().size(), is(1));
    assertThat(localMerge.nodeIds().iterator().next(), is(NODE_ID));
    assertEquals(mergePhase.outputTypes(), localMerge.inputTypes());
    assertThat(localMerge.projections(), empty());
}
Also used : MergePhase(io.crate.execution.dsl.phases.MergePhase) Merge(io.crate.planner.Merge) Collect(io.crate.planner.node.dql.Collect) EvalProjection(io.crate.execution.dsl.projection.EvalProjection) InputColumn(io.crate.expression.symbol.InputColumn) GroupProjection(io.crate.execution.dsl.projection.GroupProjection) RoutedCollectPhase(io.crate.execution.dsl.phases.RoutedCollectPhase) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest) Test(org.junit.Test) RandomizedTest(com.carrotsearch.randomizedtesting.RandomizedTest)

Example 5 with EvalProjection

use of io.crate.execution.dsl.projection.EvalProjection in project crate by crate.

the class InsertPlannerTest method testInsertFromSubQueryReduceOnCollectorGroupByWithCast.

@Test
public void testInsertFromSubQueryReduceOnCollectorGroupByWithCast() {
    Merge merge = e.plan("insert into users (id, name) (select id, count(*) from users group by id)");
    Collect nonDistributedGroupBy = (Collect) merge.subPlan();
    RoutedCollectPhase collectPhase = ((RoutedCollectPhase) nonDistributedGroupBy.collectPhase());
    assertThat(collectPhase.projections(), contains(instanceOf(GroupProjection.class), instanceOf(EvalProjection.class), instanceOf(ColumnIndexWriterProjection.class)));
    EvalProjection collectTopN = (EvalProjection) collectPhase.projections().get(1);
    assertThat(collectTopN.outputs(), contains(isInputColumn(0), isFunction(ImplicitCastFunction.NAME, List.of(DataTypes.LONG, DataTypes.STRING))));
    ColumnIndexWriterProjection columnIndexWriterProjection = (ColumnIndexWriterProjection) collectPhase.projections().get(2);
    assertThat(columnIndexWriterProjection.columnReferencesExclPartition(), contains(isReference("id"), isReference("name")));
    MergePhase mergePhase = merge.mergePhase();
    assertThat(mergePhase.projections(), contains(instanceOf(MergeCountProjection.class)));
}
Also used : MergePhase(io.crate.execution.dsl.phases.MergePhase) Collect(io.crate.planner.node.dql.Collect) EvalProjection(io.crate.execution.dsl.projection.EvalProjection) ColumnIndexWriterProjection(io.crate.execution.dsl.projection.ColumnIndexWriterProjection) RoutedCollectPhase(io.crate.execution.dsl.phases.RoutedCollectPhase) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest) Test(org.junit.Test)

Aggregations

EvalProjection (io.crate.execution.dsl.projection.EvalProjection)8 Symbol (io.crate.expression.symbol.Symbol)4 Collect (io.crate.planner.node.dql.Collect)4 MergePhase (io.crate.execution.dsl.phases.MergePhase)3 RoutedCollectPhase (io.crate.execution.dsl.phases.RoutedCollectPhase)3 InputColumn (io.crate.expression.symbol.InputColumn)3 CrateDummyClusterServiceUnitTest (io.crate.test.integration.CrateDummyClusterServiceUnitTest)3 ArrayList (java.util.ArrayList)3 Test (org.junit.Test)3 RandomizedTest (com.carrotsearch.randomizedtesting.RandomizedTest)2 ColumnIndexWriterProjection (io.crate.execution.dsl.projection.ColumnIndexWriterProjection)2 GroupProjection (io.crate.execution.dsl.projection.GroupProjection)2 InputColumns (io.crate.execution.dsl.projection.builder.InputColumns)2 SelectSymbol (io.crate.expression.symbol.SelectSymbol)2 Reference (io.crate.metadata.Reference)2 Merge (io.crate.planner.Merge)2 OrderBy (io.crate.analyze.OrderBy)1 AbstractTableRelation (io.crate.analyze.relations.AbstractTableRelation)1 DocTableRelation (io.crate.analyze.relations.DocTableRelation)1 DocKeys (io.crate.analyze.where.DocKeys)1