Examples with RoutedCollectPhase io.crate.planner.node.dql.RoutedCollectPhase used on opensource projects

Example 16 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class RemoteCollectorFactory method createCollector.

/**
     * create a RemoteCollector
     * The RemoteCollector will collect data from another node using a wormhole as if it was collecting on this node.
     * <p>
     * This should only be used if a shard is not available on the current node due to a relocation
     */
public CrateCollector.Builder createCollector(String index, Integer shardId, RoutedCollectPhase collectPhase, final RamAccountingContext ramAccountingContext) {
    // new job because subContexts can't be merged into an existing job
    final UUID childJobId = UUID.randomUUID();
    IndexShardRoutingTable shardRoutings = clusterService.state().routingTable().shardRoutingTable(index, shardId);
    // for update operations primaryShards must be used
    // (for others that wouldn't be the case, but at this point it is not easily visible which is the case)
    ShardRouting shardRouting = shardRoutings.primaryShard();
    final String remoteNodeId = shardRouting.currentNodeId();
    assert remoteNodeId != null : "primaryShard not assigned :(";
    final String localNodeId = clusterService.localNode().getId();
    final RoutedCollectPhase newCollectPhase = createNewCollectPhase(childJobId, collectPhase, index, shardId, remoteNodeId);
    return consumer -> new RemoteCollector(childJobId, localNodeId, remoteNodeId, transportActionProvider.transportJobInitAction(), transportActionProvider.transportKillJobsNodeAction(), jobContextService, ramAccountingContext, consumer, newCollectPhase);
}

Example 17 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class SystemCollectSource method getCollector.

@Override
public CrateCollector getCollector(CollectPhase phase, BatchConsumer consumer, JobCollectContext jobCollectContext) {
    RoutedCollectPhase collectPhase = (RoutedCollectPhase) phase;
    // sys.operations can contain a _node column - these refs need to be normalized into literals
    EvaluatingNormalizer normalizer = new EvaluatingNormalizer(functions, RowGranularity.DOC, ReplaceMode.COPY, new NodeSysReferenceResolver(nodeSysExpression), null);
    final RoutedCollectPhase routedCollectPhase = collectPhase.normalize(normalizer, null);
    Map<String, Map<String, List<Integer>>> locations = collectPhase.routing().locations();
    String table = Iterables.getOnlyElement(locations.get(clusterService.localNode().getId()).keySet());
    Supplier<CompletableFuture<? extends Iterable<?>>> iterableGetter = iterableGetters.get(table);
    assert iterableGetter != null : "iterableGetter for " + table + " must exist";
    boolean requiresScroll = consumer.requiresScroll();
    return BatchIteratorCollectorBridge.newInstance(() -> iterableGetter.get().thenApply(dataIterable -> RowsBatchIterator.newInstance(dataIterableToRowsIterable(routedCollectPhase, requiresScroll, dataIterable), collectPhase.toCollect().size())), consumer);
}

Example 18 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class DeleteStatementPlanner method collectWithDeleteProjection.

private static Plan collectWithDeleteProjection(TableInfo tableInfo, WhereClause whereClause, Planner.Context plannerContext) {
    // for delete, we always need to collect the `_uid`
    Reference idReference = tableInfo.getReference(DocSysColumns.ID);
    DeleteProjection deleteProjection = new DeleteProjection(new InputColumn(0, DataTypes.STRING));
    Routing routing = plannerContext.allocateRouting(tableInfo, whereClause, Preference.PRIMARY.type());
    RoutedCollectPhase collectPhase = new RoutedCollectPhase(plannerContext.jobId(), plannerContext.nextExecutionPhaseId(), "collect", routing, tableInfo.rowGranularity(), ImmutableList.of(idReference), ImmutableList.of(deleteProjection), whereClause, DistributionInfo.DEFAULT_BROADCAST);
    Collect collect = new Collect(collectPhase, TopN.NO_LIMIT, 0, 1, 1, null);
    return Merge.ensureOnHandler(collect, plannerContext, Collections.singletonList(MergeCountProjection.INSTANCE));
}

Example 19 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class InsertPlannerTest method testInsertFromSubQueryReduceOnCollectorGroupByWithCast.

@Test
public void testInsertFromSubQueryReduceOnCollectorGroupByWithCast() throws Exception {
    Merge merge = e.plan("insert into users (id, name) (select id, count(*) from users group by id)");
    Collect nonDistributedGroupBy = (Collect) merge.subPlan();
    RoutedCollectPhase collectPhase = ((RoutedCollectPhase) nonDistributedGroupBy.collectPhase());
    assertThat(collectPhase.projections(), contains(instanceOf(GroupProjection.class), instanceOf(EvalProjection.class), instanceOf(ColumnIndexWriterProjection.class)));
    EvalProjection collectTopN = (EvalProjection) collectPhase.projections().get(1);
    assertThat(collectTopN.outputs(), contains(isInputColumn(0), isFunction("to_string")));
    ColumnIndexWriterProjection columnIndexWriterProjection = (ColumnIndexWriterProjection) collectPhase.projections().get(2);
    assertThat(columnIndexWriterProjection.columnReferences(), contains(isReference("id"), isReference("name")));
    MergePhase mergePhase = merge.mergePhase();
    assertThat(mergePhase.projections(), contains(instanceOf(MergeCountProjection.class)));
}

Example 20 with RoutedCollectPhase

use of io.crate.planner.node.dql.RoutedCollectPhase in project crate by crate.

the class InsertPlannerTest method testInsertFromQueryWithPartitionedColumn.

@Test
public void testInsertFromQueryWithPartitionedColumn() throws Exception {
    Merge planNode = e.plan("insert into users (id, date) (select id, date from parted_pks)");
    Collect queryAndFetch = (Collect) planNode.subPlan();
    RoutedCollectPhase collectPhase = ((RoutedCollectPhase) queryAndFetch.collectPhase());
    List<Symbol> toCollect = collectPhase.toCollect();
    assertThat(toCollect.size(), is(2));
    assertThat(toCollect.get(0), isFunction("to_long"));
    assertThat(((Function) toCollect.get(0)).arguments().get(0), isReference("_doc['id']"));
    assertThat((Reference) toCollect.get(1), equalTo(new Reference(new ReferenceIdent(TableDefinitions.PARTED_PKS_IDENT, "date"), RowGranularity.PARTITION, DataTypes.TIMESTAMP)));
}