Examples with Projection io.crate.execution.dsl.projection.Projection used on opensource projects

Example 16 with Projection

use of io.crate.execution.dsl.projection.Projection in project crate by crate.

the class GroupByPlannerTest method testNestedGroupByAggregation.

@Test
public void testNestedGroupByAggregation() throws Exception {
    var e = SQLExecutor.builder(clusterService, 2, RandomizedTest.getRandom(), List.of()).build();
    Collect collect = e.plan("select count(*) from (" + "  select max(load['1']) as maxLoad, hostname " + "  from sys.nodes " + "  group by hostname having max(load['1']) > 50) as nodes " + "group by hostname");
    assertThat("would require merge if more than 1 nodeIds", collect.nodeIds().size(), is(1));
    CollectPhase collectPhase = collect.collectPhase();
    assertThat(collectPhase.projections(), contains(instanceOf(GroupProjection.class), instanceOf(FilterProjection.class), instanceOf(EvalProjection.class), instanceOf(GroupProjection.class), instanceOf(EvalProjection.class)));
    Projection firstGroupProjection = collectPhase.projections().get(0);
    assertThat(((GroupProjection) firstGroupProjection).mode(), is(AggregateMode.ITER_FINAL));
    Projection secondGroupProjection = collectPhase.projections().get(3);
    assertThat(((GroupProjection) secondGroupProjection).mode(), is(AggregateMode.ITER_FINAL));
}

Example 17 with Projection

use of io.crate.execution.dsl.projection.Projection in project crate by crate.

the class GroupByPlannerTest method testGroupByWithHavingAndLimit.

@Test
public void testGroupByWithHavingAndLimit() throws Exception {
    var e = SQLExecutor.builder(clusterService, 2, RandomizedTest.getRandom(), List.of()).addTable(TableDefinitions.USER_TABLE_DEFINITION).build();
    Merge planNode = e.plan("select count(*), name from users group by name having count(*) > 1 limit 100");
    Merge reducerMerge = (Merge) planNode.subPlan();
    // reducer
    MergePhase mergePhase = reducerMerge.mergePhase();
    Projection projection = mergePhase.projections().get(1);
    assertThat(projection, instanceOf(FilterProjection.class));
    FilterProjection filterProjection = (FilterProjection) projection;
    Symbol countArgument = ((Function) filterProjection.query()).arguments().get(0);
    assertThat(countArgument, instanceOf(InputColumn.class));
    // pointing to second output from group projection
    assertThat(((InputColumn) countArgument).index(), is(1));
    // outputs: name, count(*)
    TopNProjection topN = (TopNProjection) mergePhase.projections().get(2);
    assertThat(topN.outputs().get(0).valueType(), Is.is(DataTypes.STRING));
    assertThat(topN.outputs().get(1).valueType(), Is.<DataType>is(DataTypes.LONG));
    MergePhase localMerge = planNode.mergePhase();
    // topN projection
    // outputs: count(*), name
    topN = (TopNProjection) localMerge.projections().get(0);
    assertThat(topN.outputs().get(0).valueType(), Is.<DataType>is(DataTypes.LONG));
    assertThat(topN.outputs().get(1).valueType(), Is.<DataType>is(DataTypes.STRING));
}

Example 18 with Projection

use of io.crate.execution.dsl.projection.Projection in project crate by crate.

the class GroupByPlannerTest method testCountDistinctWithGroupBy.

@Test
public void testCountDistinctWithGroupBy() throws Exception {
    var e = SQLExecutor.builder(clusterService, 2, RandomizedTest.getRandom(), List.of()).addTable(TableDefinitions.USER_TABLE_DEFINITION).build();
    Merge distributedGroupByMerge = e.plan("select count(distinct id), name from users group by name order by count(distinct id)");
    Merge reducerMerge = (Merge) distributedGroupByMerge.subPlan();
    CollectPhase collectPhase = ((Collect) reducerMerge.subPlan()).collectPhase();
    // collect
    assertThat(collectPhase.toCollect().get(0), instanceOf(Reference.class));
    assertThat(collectPhase.toCollect().size(), is(2));
    assertThat(((Reference) collectPhase.toCollect().get(0)).column().name(), is("id"));
    assertThat(((Reference) collectPhase.toCollect().get(1)).column().name(), is("name"));
    Projection projection = collectPhase.projections().get(0);
    assertThat(projection, instanceOf(GroupProjection.class));
    GroupProjection groupProjection = (GroupProjection) projection;
    Symbol groupKey = groupProjection.keys().get(0);
    assertThat(groupKey, instanceOf(InputColumn.class));
    assertThat(((InputColumn) groupKey).index(), is(1));
    assertThat(groupProjection.values().size(), is(1));
    assertThat(groupProjection.mode(), is(AggregateMode.ITER_PARTIAL));
    Aggregation aggregation = groupProjection.values().get(0);
    Symbol aggregationInput = aggregation.inputs().get(0);
    assertThat(aggregationInput.symbolType(), is(SymbolType.INPUT_COLUMN));
    // reducer
    MergePhase mergePhase = reducerMerge.mergePhase();
    assertThat(mergePhase.projections(), contains(instanceOf(GroupProjection.class), instanceOf(OrderedTopNProjection.class), instanceOf(EvalProjection.class)));
    Projection groupProjection1 = mergePhase.projections().get(0);
    groupProjection = (GroupProjection) groupProjection1;
    assertThat(groupProjection.keys().get(0), instanceOf(InputColumn.class));
    assertThat(((InputColumn) groupProjection.keys().get(0)).index(), is(0));
    assertThat(groupProjection.mode(), is(AggregateMode.PARTIAL_FINAL));
    assertThat(groupProjection.values().get(0), instanceOf(Aggregation.class));
    OrderedTopNProjection topNProjection = (OrderedTopNProjection) mergePhase.projections().get(1);
    Symbol collection_count = topNProjection.outputs().get(0);
    assertThat(collection_count, SymbolMatchers.isInputColumn(0));
    // handler
    MergePhase localMergeNode = distributedGroupByMerge.mergePhase();
    assertThat(localMergeNode.projections(), Matchers.emptyIterable());
}

Example 19 with Projection

use of io.crate.execution.dsl.projection.Projection in project crate by crate.

the class MergeNodeTest method testSerialization.

@Test
public void testSerialization() throws Exception {
    List<Symbol> keys = Collections.singletonList(new InputColumn(0, DataTypes.STRING));
    List<Aggregation> aggregations = Collections.singletonList(new Aggregation(CountAggregation.COUNT_STAR_SIGNATURE, CountAggregation.COUNT_STAR_SIGNATURE.getReturnType().createType(), Collections.emptyList()));
    GroupProjection groupProjection = new GroupProjection(keys, aggregations, AggregateMode.PARTIAL_FINAL, RowGranularity.CLUSTER);
    TopNProjection topNProjection = new TopNProjection(10, 0, Symbols.typeView(groupProjection.outputs()));
    List<Projection> projections = Arrays.asList(groupProjection, topNProjection);
    MergePhase node = new MergePhase(UUID.randomUUID(), 0, "merge", 2, 1, Set.of("node1", "node2"), List.of(DataTypes.UNDEFINED, DataTypes.STRING), projections, DistributionInfo.DEFAULT_BROADCAST, null);
    BytesStreamOutput output = new BytesStreamOutput();
    node.writeTo(output);
    StreamInput input = output.bytes().streamInput();
    MergePhase node2 = new MergePhase(input);
    assertThat(node.numUpstreams(), is(node2.numUpstreams()));
    assertThat(node.nodeIds(), is(node2.nodeIds()));
    assertThat(node.jobId(), is(node2.jobId()));
    assertEquals(node.inputTypes(), node2.inputTypes());
    assertThat(node.phaseId(), is(node2.phaseId()));
    assertThat(node.distributionInfo(), is(node2.distributionInfo()));
}

Example 20 with Projection

use of io.crate.execution.dsl.projection.Projection in project crate by crate.

the class CopyFromPlan method planCopyFromExecution.

public static ExecutionPlan planCopyFromExecution(AnalyzedCopyFrom copyFrom, DiscoveryNodes allNodes, PlannerContext context, Row params, SubQueryResults subQueryResults) {
    var boundedCopyFrom = bind(copyFrom, context.transactionContext(), context.nodeContext(), params, subQueryResults);
    /*
         * Create a plan that reads json-objects-lines from a file
         * and then executes upsert requests to index the data
         */
    DocTableInfo table = boundedCopyFrom.tableInfo();
    String partitionIdent = boundedCopyFrom.partitionIdent();
    List<String> partitionedByNames = Collections.emptyList();
    List<String> partitionValues = Collections.emptyList();
    if (partitionIdent == null) {
        if (table.isPartitioned()) {
            partitionedByNames = Lists2.map(table.partitionedBy(), ColumnIdent::fqn);
        }
    } else {
        assert table.isPartitioned() : "table must be partitioned if partitionIdent is set";
        // partitionIdent is present -> possible to index raw source into concrete es index
        partitionValues = PartitionName.decodeIdent(partitionIdent);
    }
    // need to exclude _id columns; they're auto generated and won't be available in the files being imported
    ColumnIdent clusteredBy = table.clusteredBy();
    if (DocSysColumns.ID.equals(clusteredBy)) {
        clusteredBy = null;
    }
    List<Reference> primaryKeyRefs = table.primaryKey().stream().filter(r -> !r.equals(DocSysColumns.ID)).map(table::getReference).collect(Collectors.toList());
    List<Symbol> toCollect = getSymbolsRequiredForShardIdCalc(primaryKeyRefs, table.partitionedByColumns(), clusteredBy == null ? null : table.getReference(clusteredBy));
    Reference rawOrDoc = rawOrDoc(table, partitionIdent);
    final int rawOrDocIdx = toCollect.size();
    toCollect.add(rawOrDoc);
    String[] excludes = partitionedByNames.size() > 0 ? partitionedByNames.toArray(new String[0]) : null;
    InputColumns.SourceSymbols sourceSymbols = new InputColumns.SourceSymbols(toCollect);
    Symbol clusteredByInputCol = null;
    if (clusteredBy != null) {
        clusteredByInputCol = InputColumns.create(table.getReference(clusteredBy), sourceSymbols);
    }
    SourceIndexWriterProjection sourceIndexWriterProjection;
    List<? extends Symbol> projectionOutputs = AbstractIndexWriterProjection.OUTPUTS;
    boolean returnSummary = copyFrom instanceof AnalyzedCopyFromReturnSummary;
    boolean failFast = boundedCopyFrom.settings().getAsBoolean("fail_fast", false);
    if (returnSummary || failFast) {
        final InputColumn sourceUriSymbol = new InputColumn(toCollect.size(), DataTypes.STRING);
        toCollect.add(SourceUriExpression.getReferenceForRelation(table.ident()));
        final InputColumn sourceUriFailureSymbol = new InputColumn(toCollect.size(), DataTypes.STRING);
        toCollect.add(SourceUriFailureExpression.getReferenceForRelation(table.ident()));
        final InputColumn lineNumberSymbol = new InputColumn(toCollect.size(), DataTypes.LONG);
        toCollect.add(SourceLineNumberExpression.getReferenceForRelation(table.ident()));
        if (returnSummary) {
            List<? extends Symbol> fields = ((AnalyzedCopyFromReturnSummary) copyFrom).outputs();
            projectionOutputs = InputColumns.create(fields, new InputColumns.SourceSymbols(fields));
        }
        sourceIndexWriterProjection = new SourceIndexWriterReturnSummaryProjection(table.ident(), partitionIdent, table.getReference(DocSysColumns.RAW), new InputColumn(rawOrDocIdx, rawOrDoc.valueType()), table.primaryKey(), InputColumns.create(table.partitionedByColumns(), sourceSymbols), clusteredBy, boundedCopyFrom.settings(), null, excludes, InputColumns.create(primaryKeyRefs, sourceSymbols), clusteredByInputCol, projectionOutputs, // autoCreateIndices
        table.isPartitioned(), sourceUriSymbol, sourceUriFailureSymbol, lineNumberSymbol);
    } else {
        sourceIndexWriterProjection = new SourceIndexWriterProjection(table.ident(), partitionIdent, table.getReference(DocSysColumns.RAW), new InputColumn(rawOrDocIdx, rawOrDoc.valueType()), table.primaryKey(), InputColumns.create(table.partitionedByColumns(), sourceSymbols), clusteredBy, boundedCopyFrom.settings(), null, excludes, InputColumns.create(primaryKeyRefs, sourceSymbols), clusteredByInputCol, projectionOutputs, // autoCreateIndices
        table.isPartitioned());
    }
    // the partitionedBy-inputColumns created for the projection are still valid because the positions are not changed
    if (partitionValues != null) {
        rewriteToCollectToUsePartitionValues(table.partitionedByColumns(), partitionValues, toCollect);
    }
    FileUriCollectPhase collectPhase = new FileUriCollectPhase(context.jobId(), context.nextExecutionPhaseId(), "copyFrom", getExecutionNodes(allNodes, boundedCopyFrom.settings().getAsInt("num_readers", allNodes.getSize()), boundedCopyFrom.nodePredicate()), boundedCopyFrom.uri(), toCollect, Collections.emptyList(), boundedCopyFrom.settings().get("compression", null), boundedCopyFrom.settings().getAsBoolean("shared", null), CopyFromParserProperties.of(boundedCopyFrom.settings()), boundedCopyFrom.inputFormat(), boundedCopyFrom.settings());
    Collect collect = new Collect(collectPhase, TopN.NO_LIMIT, 0, 1, -1, null);
    // add the projection to the plan to ensure that the outputs are correctly set to the projection outputs
    collect.addProjection(sourceIndexWriterProjection);
    List<Projection> handlerProjections;
    if (returnSummary) {
        handlerProjections = Collections.emptyList();
    } else {
        handlerProjections = List.of(MergeCountProjection.INSTANCE);
    }
    return Merge.ensureOnHandler(collect, context, handlerProjections);
}