Search in sources :

Example 1 with IntObjectCursor

use of com.carrotsearch.hppc.cursors.IntObjectCursor in project lucene-solr by apache.

the class ExpandComponent method getGroupQuery.

private Query getGroupQuery(String fname, int size, IntObjectHashMap<BytesRef> ordBytes) throws Exception {
    BytesRef[] bytesRefs = new BytesRef[size];
    int index = -1;
    Iterator<IntObjectCursor<BytesRef>> it = ordBytes.iterator();
    while (it.hasNext()) {
        IntObjectCursor<BytesRef> cursor = it.next();
        bytesRefs[++index] = cursor.value;
    }
    return new SolrConstantScoreQuery(new QueryWrapperFilter(new TermInSetQuery(fname, bytesRefs)));
}
Also used : IntObjectCursor(com.carrotsearch.hppc.cursors.IntObjectCursor) TermInSetQuery(org.apache.lucene.search.TermInSetQuery) SolrConstantScoreQuery(org.apache.solr.search.SolrConstantScoreQuery) QueryWrapperFilter(org.apache.solr.search.QueryWrapperFilter) BytesRef(org.apache.lucene.util.BytesRef)

Example 2 with IntObjectCursor

use of com.carrotsearch.hppc.cursors.IntObjectCursor in project crate by crate.

the class NodeFetchOperation method doFetch.

private CompletableFuture<? extends IntObjectMap<StreamBucket>> doFetch(FetchTask fetchTask, IntObjectMap<IntArrayList> toFetch) throws Exception {
    HashMap<RelationName, TableFetchInfo> tableFetchInfos = getTableFetchInfos(fetchTask);
    // RamAccounting is per doFetch call instead of per FetchTask/fetchPhase
    // To be able to free up the memory count when the operation is complete
    final var ramAccounting = ConcurrentRamAccounting.forCircuitBreaker("fetch-" + fetchTask.id(), circuitBreaker);
    ArrayList<Supplier<StreamBucket>> collectors = new ArrayList<>(toFetch.size());
    for (IntObjectCursor<IntArrayList> toFetchCursor : toFetch) {
        final int readerId = toFetchCursor.key;
        final IntArrayList docIds = toFetchCursor.value;
        RelationName ident = fetchTask.tableIdent(readerId);
        final TableFetchInfo tfi = tableFetchInfos.get(ident);
        assert tfi != null : "tfi must not be null";
        var collector = tfi.createCollector(readerId, new BlockBasedRamAccounting(ramAccounting::addBytes, BlockBasedRamAccounting.MAX_BLOCK_SIZE_IN_BYTES));
        collectors.add(() -> collector.collect(docIds));
    }
    return ThreadPools.runWithAvailableThreads(executor, ThreadPools.numIdleThreads(executor, numProcessors), collectors).thenApply(buckets -> {
        var toFetchIt = toFetch.iterator();
        assert toFetch.size() == buckets.size() : "Must have a bucket per reader and they must be in the same order";
        IntObjectHashMap<StreamBucket> bucketByReader = new IntObjectHashMap<>(toFetch.size());
        for (var bucket : buckets) {
            assert toFetchIt.hasNext() : "toFetchIt must have an element if there is one in buckets";
            int readerId = toFetchIt.next().key;
            bucketByReader.put(readerId, bucket);
        }
        return bucketByReader;
    }).whenComplete((result, err) -> ramAccounting.close());
}
Also used : IntObjectCursor(com.carrotsearch.hppc.cursors.IntObjectCursor) StreamBucket(io.crate.execution.engine.distribution.StreamBucket) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) RelationName(io.crate.metadata.RelationName) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RootTask(io.crate.execution.jobs.RootTask) Supplier(java.util.function.Supplier) JobsLogs(io.crate.execution.engine.collect.stats.JobsLogs) ArrayList(java.util.ArrayList) BlockBasedRamAccounting(io.crate.breaker.BlockBasedRamAccounting) IntArrayList(com.carrotsearch.hppc.IntArrayList) Symbols(io.crate.expression.symbol.Symbols) Map(java.util.Map) BiConsumer(java.util.function.BiConsumer) CircuitBreaker(org.elasticsearch.common.breaker.CircuitBreaker) Nullable(javax.annotation.Nullable) LuceneReferenceResolver(io.crate.expression.reference.doc.lucene.LuceneReferenceResolver) Streamer(io.crate.Streamer) Collection(java.util.Collection) IndexService(org.elasticsearch.index.IndexService) Reference(io.crate.metadata.Reference) UUID(java.util.UUID) RamAccounting(io.crate.breaker.RamAccounting) LuceneCollectorExpression(io.crate.expression.reference.doc.lucene.LuceneCollectorExpression) TasksService(io.crate.execution.jobs.TasksService) IntObjectMap(com.carrotsearch.hppc.IntObjectMap) ConcurrentRamAccounting(io.crate.breaker.ConcurrentRamAccounting) IntObjectHashMap(com.carrotsearch.hppc.IntObjectHashMap) SQLExceptions(io.crate.exceptions.SQLExceptions) ThreadPools(io.crate.execution.support.ThreadPools) IntObjectHashMap(com.carrotsearch.hppc.IntObjectHashMap) ArrayList(java.util.ArrayList) IntArrayList(com.carrotsearch.hppc.IntArrayList) BlockBasedRamAccounting(io.crate.breaker.BlockBasedRamAccounting) RelationName(io.crate.metadata.RelationName) Supplier(java.util.function.Supplier) IntArrayList(com.carrotsearch.hppc.IntArrayList)

Aggregations

IntObjectCursor (com.carrotsearch.hppc.cursors.IntObjectCursor)2 IntArrayList (com.carrotsearch.hppc.IntArrayList)1 IntObjectHashMap (com.carrotsearch.hppc.IntObjectHashMap)1 IntObjectMap (com.carrotsearch.hppc.IntObjectMap)1 Streamer (io.crate.Streamer)1 BlockBasedRamAccounting (io.crate.breaker.BlockBasedRamAccounting)1 ConcurrentRamAccounting (io.crate.breaker.ConcurrentRamAccounting)1 RamAccounting (io.crate.breaker.RamAccounting)1 SQLExceptions (io.crate.exceptions.SQLExceptions)1 JobsLogs (io.crate.execution.engine.collect.stats.JobsLogs)1 StreamBucket (io.crate.execution.engine.distribution.StreamBucket)1 RootTask (io.crate.execution.jobs.RootTask)1 TasksService (io.crate.execution.jobs.TasksService)1 ThreadPools (io.crate.execution.support.ThreadPools)1 LuceneCollectorExpression (io.crate.expression.reference.doc.lucene.LuceneCollectorExpression)1 LuceneReferenceResolver (io.crate.expression.reference.doc.lucene.LuceneReferenceResolver)1 Symbols (io.crate.expression.symbol.Symbols)1 Reference (io.crate.metadata.Reference)1 RelationName (io.crate.metadata.RelationName)1 ArrayList (java.util.ArrayList)1