Examples with SharedShardContext io.crate.action.job.SharedShardContext used on opensource projects

Example 1 with SharedShardContext

use of io.crate.action.job.SharedShardContext in project crate by crate.

the class ShardCollectSource method createMultiShardScoreDocCollector.

private CrateCollector createMultiShardScoreDocCollector(RoutedCollectPhase collectPhase, BatchConsumer consumer, JobCollectContext jobCollectContext, String localNodeId) {
    Map<String, Map<String, List<Integer>>> locations = collectPhase.routing().locations();
    SharedShardContexts sharedShardContexts = jobCollectContext.sharedShardContexts();
    Map<String, List<Integer>> indexShards = locations.get(localNodeId);
    List<OrderedDocCollector> orderedDocCollectors = new ArrayList<>();
    for (Map.Entry<String, List<Integer>> entry : indexShards.entrySet()) {
        String indexName = entry.getKey();
        for (Integer shardNum : entry.getValue()) {
            ShardId shardId = new ShardId(indexName, shardNum);
            SharedShardContext context = sharedShardContexts.getOrCreateContext(shardId);
            try {
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                orderedDocCollectors.add(shardCollectorProvider.getOrderedCollector(collectPhase, context, jobCollectContext, consumer.requiresScroll()));
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                throw e;
            } catch (IndexNotFoundException e) {
                if (PartitionName.isPartition(indexName)) {
                    break;
                }
                throw e;
            } catch (Throwable t) {
                throw new UnhandledServerException(t);
            }
        }
    }
    OrderBy orderBy = collectPhase.orderBy();
    assert orderBy != null : "orderBy must not be null";
    return BatchIteratorCollectorBridge.newInstance(OrderedLuceneBatchIteratorFactory.newInstance(orderedDocCollectors, collectPhase.toCollect().size(), OrderingByPosition.rowOrdering(OrderByPositionVisitor.orderByPositions(orderBy.orderBySymbols(), collectPhase.toCollect()), orderBy.reverseFlags(), orderBy.nullsFirst()), executor, consumer.requiresScroll()), consumer);
}

Example 2 with SharedShardContext

use of io.crate.action.job.SharedShardContext in project crate by crate.

the class LuceneShardCollectorProvider method getBuilder.

@Override
protected CrateCollector.Builder getBuilder(RoutedCollectPhase collectPhase, boolean requiresScroll, JobCollectContext jobCollectContext) {
    SharedShardContext sharedShardContext = jobCollectContext.sharedShardContexts().getOrCreateContext(indexShard.shardId());
    Engine.Searcher searcher = sharedShardContext.acquireSearcher();
    IndexShard indexShard = sharedShardContext.indexShard();
    try {
        LuceneQueryBuilder.Context queryContext = luceneQueryBuilder.convert(collectPhase.whereClause(), indexShard.mapperService(), indexShard.indexFieldDataService(), indexShard.indexService().cache());
        jobCollectContext.addSearcher(sharedShardContext.readerId(), searcher);
        InputFactory.Context<? extends LuceneCollectorExpression<?>> docCtx = docInputFactory.extractImplementations(collectPhase);
        return new CrateDocCollectorBuilder(searcher.searcher(), queryContext.query(), queryContext.minScore(), Symbols.containsColumn(collectPhase.toCollect(), DocSysColumns.SCORE), getCollectorContext(sharedShardContext.readerId(), docCtx), jobCollectContext.queryPhaseRamAccountingContext(), docCtx.topLevelInputs(), docCtx.expressions());
    } catch (Throwable t) {
        searcher.close();
        throw t;
    }
}

Example 3 with SharedShardContext

use of io.crate.action.job.SharedShardContext in project crate by crate.

the class FetchContext method innerPrepare.

@Override
public void innerPrepare() {
    HashMap<String, TableIdent> index2TableIdent = new HashMap<>();
    for (Map.Entry<TableIdent, Collection<String>> entry : phase.tableIndices().asMap().entrySet()) {
        for (String indexName : entry.getValue()) {
            index2TableIdent.put(indexName, entry.getKey());
        }
    }
    Set<TableIdent> tablesWithFetchRefs = new HashSet<>();
    for (Reference reference : phase.fetchRefs()) {
        tablesWithFetchRefs.add(reference.ident().tableIdent());
    }
    for (Routing routing : routingIterable) {
        Map<String, Map<String, List<Integer>>> locations = routing.locations();
        Map<String, List<Integer>> indexShards = locations.get(localNodeId);
        for (Map.Entry<String, List<Integer>> indexShardsEntry : indexShards.entrySet()) {
            String index = indexShardsEntry.getKey();
            Integer base = phase.bases().get(index);
            if (base == null) {
                continue;
            }
            TableIdent ident = index2TableIdent.get(index);
            assert ident != null : "no tableIdent found for index " + index;
            tableIdents.put(base, ident);
            toFetch.put(ident, new ArrayList<Reference>());
            for (Integer shard : indexShardsEntry.getValue()) {
                ShardId shardId = new ShardId(index, shard);
                int readerId = base + shardId.id();
                SharedShardContext shardContext = shardContexts.get(readerId);
                if (shardContext == null) {
                    shardContext = sharedShardContexts.createContext(shardId, readerId);
                    shardContexts.put(readerId, shardContext);
                    if (tablesWithFetchRefs.contains(ident)) {
                        try {
                            searchers.put(readerId, shardContext.acquireSearcher());
                        } catch (IndexNotFoundException e) {
                            if (!PartitionName.isPartition(index)) {
                                throw e;
                            }
                        }
                    }
                }
            }
        }
    }
    for (Reference reference : phase.fetchRefs()) {
        Collection<Reference> references = toFetch.get(reference.ident().tableIdent());
        if (references != null) {
            references.add(reference);
        }
    }
}