Examples with Projectors io.crate.execution.engine.pipeline.Projectors used on opensource projects

Example 1 with Projectors

use of io.crate.execution.engine.pipeline.Projectors in project crate by crate.

the class PKLookupOperation method lookup.

public BatchIterator<Row> lookup(UUID jobId, TransactionContext txnCtx, Supplier<RamAccounting> ramAccountingSupplier, Supplier<MemoryManager> memoryManagerSupplier, boolean ignoreMissing, Map<ShardId, List<PKAndVersion>> idsByShard, Collection<? extends Projection> projections, boolean requiresScroll, Function<Doc, Row> resultToRow) {
    ArrayList<BatchIterator<Row>> iterators = new ArrayList<>(idsByShard.size());
    for (Map.Entry<ShardId, List<PKAndVersion>> idsByShardEntry : idsByShard.entrySet()) {
        ShardId shardId = idsByShardEntry.getKey();
        IndexService indexService = indicesService.indexService(shardId.getIndex());
        if (indexService == null) {
            if (ignoreMissing) {
                continue;
            }
            throw new IndexNotFoundException(shardId.getIndex());
        }
        IndexShard shard = indexService.getShardOrNull(shardId.id());
        if (shard == null) {
            if (ignoreMissing) {
                continue;
            }
            throw new ShardNotFoundException(shardId);
        }
        Stream<Row> rowStream = idsByShardEntry.getValue().stream().map(pkAndVersion -> lookupDoc(shard, pkAndVersion.id(), pkAndVersion.version(), VersionType.EXTERNAL, pkAndVersion.seqNo(), pkAndVersion.primaryTerm())).filter(Objects::nonNull).map(resultToRow);
        if (projections.isEmpty()) {
            final Iterable<Row> rowIterable = requiresScroll ? rowStream.map(row -> new RowN(row.materialize())).collect(Collectors.toList()) : rowStream::iterator;
            iterators.add(InMemoryBatchIterator.of(rowIterable, SentinelRow.SENTINEL, true));
        } else {
            ProjectorFactory projectorFactory;
            try {
                projectorFactory = shardCollectSource.getProjectorFactory(shardId);
            } catch (ShardNotFoundException e) {
                if (ignoreMissing) {
                    continue;
                }
                throw e;
            }
            Projectors projectors = new Projectors(projections, jobId, txnCtx, ramAccountingSupplier.get(), memoryManagerSupplier.get(), projectorFactory);
            final Iterable<Row> rowIterable = requiresScroll && !projectors.providesIndependentScroll() ? rowStream.map(row -> new RowN(row.materialize())).collect(Collectors.toList()) : rowStream::iterator;
            iterators.add(projectors.wrap(InMemoryBatchIterator.of(rowIterable, SentinelRow.SENTINEL, true)));
        }
    }
    // noinspection unchecked
    return CompositeBatchIterator.seqComposite(iterators.toArray(new BatchIterator[0]));
}

Example 2 with Projectors

use of io.crate.execution.engine.pipeline.Projectors in project crate by crate.

the class ShardCollectSource method getIterator.

@Override
public CompletableFuture<BatchIterator<Row>> getIterator(TransactionContext txnCtx, CollectPhase phase, CollectTask collectTask, boolean supportMoveToStart) {
    RoutedCollectPhase collectPhase = (RoutedCollectPhase) phase;
    String localNodeId = clusterService.localNode().getId();
    Projectors projectors = new Projectors(collectPhase.projections(), collectPhase.jobId(), collectTask.txnCtx(), collectTask.getRamAccounting(), collectTask.memoryManager(), sharedProjectorFactory);
    boolean requireMoveToStartSupport = supportMoveToStart && !projectors.providesIndependentScroll();
    if (collectPhase.maxRowGranularity() == RowGranularity.SHARD) {
        return CompletableFuture.completedFuture(projectors.wrap(InMemoryBatchIterator.of(getShardsIterator(collectTask.txnCtx(), collectPhase, localNodeId), SentinelRow.SENTINEL, true)));
    }
    OrderBy orderBy = collectPhase.orderBy();
    if (collectPhase.maxRowGranularity() == RowGranularity.DOC && orderBy != null) {
        return createMultiShardScoreDocCollector(collectPhase, requireMoveToStartSupport, collectTask, localNodeId).thenApply(projectors::wrap);
    }
    boolean hasShardProjections = Projections.hasAnyShardProjections(collectPhase.projections());
    Map<String, IntIndexedContainer> indexShards = collectPhase.routing().locations().get(localNodeId);
    List<CompletableFuture<BatchIterator<Row>>> iterators = indexShards == null ? Collections.emptyList() : getIterators(collectTask, collectPhase, requireMoveToStartSupport, indexShards);
    final CompletableFuture<BatchIterator<Row>> result;
    switch(iterators.size()) {
        case 0:
            result = CompletableFuture.completedFuture(InMemoryBatchIterator.empty(SentinelRow.SENTINEL));
            break;
        case 1:
            result = iterators.get(0);
            break;
        default:
            if (hasShardProjections) {
                // use AsyncCompositeBatchIterator for multi-threaded loadNextBatch
                // in order to process shard-based projections concurrently
                // noinspection unchecked
                result = CompletableFutures.allAsList(iterators).thenApply(its -> CompositeBatchIterator.asyncComposite(executor, availableThreads, its.toArray(new BatchIterator[0])));
            } else {
                // noinspection unchecked
                result = CompletableFutures.allAsList(iterators).thenApply(its -> CompositeBatchIterator.seqComposite(its.toArray(new BatchIterator[0])));
            }
    }
    return result.thenApply(it -> projectors.wrap(it));
}