Examples with RoutedCollectPhase io.crate.execution.dsl.phases.RoutedCollectPhase used on opensource projects

Example 1 with RoutedCollectPhase

use of io.crate.execution.dsl.phases.RoutedCollectPhase in project crate by crate.

the class Collect method executesOnShard.

@Override
public boolean executesOnShard() {
    List<Projection> projections = collectPhase.projections();
    if (projections.isEmpty()) {
        return collectPhase instanceof RoutedCollectPhase && ((RoutedCollectPhase) collectPhase).routing().containsShards();
    }
    Projection lastProjection = projections.get(projections.size() - 1);
    return lastProjection.requiredGranularity() == RowGranularity.SHARD;
}

Example 2 with RoutedCollectPhase

use of io.crate.execution.dsl.phases.RoutedCollectPhase in project crate by crate.

the class NodeStatsCollectSource method getIterator.

@Override
public CompletableFuture<BatchIterator<Row>> getIterator(TransactionContext txnCtx, CollectPhase phase, CollectTask collectTask, boolean supportMoveToStart) {
    RoutedCollectPhase collectPhase = (RoutedCollectPhase) phase;
    if (!WhereClause.canMatch(collectPhase.where())) {
        return completedFuture(InMemoryBatchIterator.empty(SentinelRow.SENTINEL));
    }
    Collection<DiscoveryNode> nodes = filterNodes(CollectionUtils.iterableAsArrayList(clusterService.state().getNodes()), collectPhase.where(), nodeCtx);
    if (nodes.isEmpty()) {
        return completedFuture(InMemoryBatchIterator.empty(SentinelRow.SENTINEL));
    }
    return completedFuture(NodeStats.newInstance(nodeStatsAction, collectPhase, nodes, txnCtx, inputFactory));
}

Example 3 with RoutedCollectPhase

use of io.crate.execution.dsl.phases.RoutedCollectPhase in project crate by crate.

the class LuceneShardCollectorProvider method getOrderedCollector.

@Override
public OrderedDocCollector getOrderedCollector(RoutedCollectPhase phase, SharedShardContext sharedShardContext, CollectTask collectTask, boolean requiresRepeat) {
    RoutedCollectPhase collectPhase = phase.normalize(shardNormalizer, collectTask.txnCtx());
    CollectorContext collectorContext;
    InputFactory.Context<? extends LuceneCollectorExpression<?>> ctx;
    var searcher = sharedShardContext.acquireSearcher("ordered-collector: " + formatSource(phase));
    collectTask.addSearcher(sharedShardContext.readerId(), searcher);
    IndexService indexService = sharedShardContext.indexService();
    QueryShardContext queryShardContext = indexService.newQueryShardContext();
    final var queryContext = luceneQueryBuilder.convert(collectPhase.where(), collectTask.txnCtx(), indexService.mapperService(), indexShard.shardId().getIndexName(), queryShardContext, table, indexService.cache());
    ctx = docInputFactory.extractImplementations(collectTask.txnCtx(), collectPhase);
    collectorContext = new CollectorContext(sharedShardContext.readerId());
    int batchSize = collectPhase.shardQueueSize(localNodeId.get());
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("[{}][{}] creating LuceneOrderedDocCollector. Expected number of rows to be collected: {}", sharedShardContext.indexShard().routingEntry().currentNodeId(), sharedShardContext.indexShard().shardId(), batchSize);
    }
    OptimizeQueryForSearchAfter optimizeQueryForSearchAfter = new OptimizeQueryForSearchAfter(collectPhase.orderBy(), queryContext.queryShardContext(), fieldTypeLookup);
    return new LuceneOrderedDocCollector(indexShard.shardId(), searcher.item(), queryContext.query(), queryContext.minScore(), Symbols.containsColumn(collectPhase.toCollect(), DocSysColumns.SCORE), batchSize, collectTask.getRamAccounting(), collectorContext, optimizeQueryForSearchAfter, LuceneSortGenerator.generateLuceneSort(collectTask.txnCtx(), collectorContext, collectPhase.orderBy(), docInputFactory, fieldTypeLookup), ctx.topLevelInputs(), ctx.expressions());
}

Example 4 with RoutedCollectPhase

use of io.crate.execution.dsl.phases.RoutedCollectPhase in project crate by crate.

the class RootTaskTest method testFailureClosesAllSubContexts.

@Test
public void testFailureClosesAllSubContexts() throws Throwable {
    String localNodeId = "localNodeId";
    RoutedCollectPhase collectPhase = Mockito.mock(RoutedCollectPhase.class);
    Routing routing = Mockito.mock(Routing.class);
    when(routing.containsShards(localNodeId)).thenReturn(false);
    when(collectPhase.phaseId()).thenReturn(1);
    when(collectPhase.routing()).thenReturn(routing);
    when(collectPhase.maxRowGranularity()).thenReturn(RowGranularity.DOC);
    RootTask.Builder builder = new RootTask.Builder(logger, UUID.randomUUID(), "dummy-user", coordinatorNode, Collections.emptySet(), mock(JobsLogs.class));
    CollectTask collectChildTask = new CollectTask(collectPhase, CoordinatorTxnCtx.systemTransactionContext(), mock(MapSideDataCollectOperation.class), RamAccounting.NO_ACCOUNTING, ramAccounting -> new OnHeapMemoryManager(ramAccounting::addBytes), new TestingRowConsumer(), mock(SharedShardContexts.class), Version.CURRENT, 4096);
    TestingRowConsumer batchConsumer = new TestingRowConsumer();
    PageBucketReceiver pageBucketReceiver = new CumulativePageBucketReceiver("n1", 2, Runnable::run, new Streamer[] { IntegerType.INSTANCE.streamer() }, batchConsumer, PassThroughPagingIterator.oneShot(), 1);
    DistResultRXTask distResultRXTask = spy(new DistResultRXTask(2, "dummy", pageBucketReceiver, RamAccounting.NO_ACCOUNTING, 1));
    builder.addTask(collectChildTask);
    builder.addTask(distResultRXTask);
    RootTask rootTask = builder.build();
    Exception failure = new Exception("failure!");
    collectChildTask.kill(failure);
    // other contexts must be killed with same failure
    verify(distResultRXTask, times(1)).kill(failure);
    assertThat(rootTask.getTask(1).completionFuture().isDone(), is(true));
    assertThat(rootTask.getTask(2).completionFuture().isDone(), is(true));
}

Example 5 with RoutedCollectPhase

use of io.crate.execution.dsl.phases.RoutedCollectPhase in project crate by crate.

the class ExecutionPhasesRootTaskTest method testGroupByServer.

@Test
public void testGroupByServer() throws Exception {
    var routingMap = new TreeMap<String, Map<String, IntIndexedContainer>>();
    routingMap.put("node1", Map.of("t1", IntArrayList.from(1, 2)));
    routingMap.put("node2", Map.of("t1", IntArrayList.from(3, 4)));
    Routing twoNodeRouting = new Routing(routingMap);
    UUID jobId = UUID.randomUUID();
    RoutedCollectPhase c1 = new RoutedCollectPhase(jobId, 1, "c1", twoNodeRouting, RowGranularity.DOC, List.of(), List.of(), WhereClause.MATCH_ALL.queryOrFallback(), DistributionInfo.DEFAULT_BROADCAST);
    MergePhase m1 = new MergePhase(jobId, 2, "merge1", 2, 1, Set.of("node3", "node4"), List.of(), List.of(), DistributionInfo.DEFAULT_BROADCAST, null);
    MergePhase m2 = new MergePhase(jobId, 3, "merge2", 2, 1, Set.of("node1", "node3"), List.of(), List.of(), DistributionInfo.DEFAULT_BROADCAST, null);
    NodeOperation n1 = NodeOperation.withDownstream(c1, m1, (byte) 0);
    NodeOperation n2 = NodeOperation.withDownstream(m1, m2, (byte) 0);
    NodeOperation n3 = NodeOperation.withDownstream(m2, mock(ExecutionPhase.class), (byte) 0);
    Map<String, Collection<NodeOperation>> groupByServer = NodeOperationGrouper.groupByServer(List.of(n1, n2, n3));
    assertThat(groupByServer.containsKey("node1"), is(true));
    assertThat(groupByServer.get("node1"), Matchers.containsInAnyOrder(n1, n3));
    assertThat(groupByServer.containsKey("node2"), is(true));
    assertThat(groupByServer.get("node2"), Matchers.containsInAnyOrder(n1));
    assertThat(groupByServer.containsKey("node3"), is(true));
    assertThat(groupByServer.get("node3"), Matchers.containsInAnyOrder(n2, n3));
    assertThat(groupByServer.containsKey("node4"), is(true));
    assertThat(groupByServer.get("node4"), Matchers.containsInAnyOrder(n2));
}