Search in sources :

Example 6 with IntObjectMap

use of com.carrotsearch.hppc.IntObjectMap in project crate by crate.

the class NodeFetchOperation method doFetch.

private CompletableFuture<? extends IntObjectMap<StreamBucket>> doFetch(FetchTask fetchTask, IntObjectMap<IntArrayList> toFetch) throws Exception {
    HashMap<RelationName, TableFetchInfo> tableFetchInfos = getTableFetchInfos(fetchTask);
    // RamAccounting is per doFetch call instead of per FetchTask/fetchPhase
    // To be able to free up the memory count when the operation is complete
    final var ramAccounting = ConcurrentRamAccounting.forCircuitBreaker("fetch-" + fetchTask.id(), circuitBreaker);
    ArrayList<Supplier<StreamBucket>> collectors = new ArrayList<>(toFetch.size());
    for (IntObjectCursor<IntArrayList> toFetchCursor : toFetch) {
        final int readerId = toFetchCursor.key;
        final IntArrayList docIds = toFetchCursor.value;
        RelationName ident = fetchTask.tableIdent(readerId);
        final TableFetchInfo tfi = tableFetchInfos.get(ident);
        assert tfi != null : "tfi must not be null";
        var collector = tfi.createCollector(readerId, new BlockBasedRamAccounting(ramAccounting::addBytes, BlockBasedRamAccounting.MAX_BLOCK_SIZE_IN_BYTES));
        collectors.add(() -> collector.collect(docIds));
    }
    return ThreadPools.runWithAvailableThreads(executor, ThreadPools.numIdleThreads(executor, numProcessors), collectors).thenApply(buckets -> {
        var toFetchIt = toFetch.iterator();
        assert toFetch.size() == buckets.size() : "Must have a bucket per reader and they must be in the same order";
        IntObjectHashMap<StreamBucket> bucketByReader = new IntObjectHashMap<>(toFetch.size());
        for (var bucket : buckets) {
            assert toFetchIt.hasNext() : "toFetchIt must have an element if there is one in buckets";
            int readerId = toFetchIt.next().key;
            bucketByReader.put(readerId, bucket);
        }
        return bucketByReader;
    }).whenComplete((result, err) -> ramAccounting.close());
}
Also used : IntObjectCursor(com.carrotsearch.hppc.cursors.IntObjectCursor) StreamBucket(io.crate.execution.engine.distribution.StreamBucket) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) RelationName(io.crate.metadata.RelationName) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RootTask(io.crate.execution.jobs.RootTask) Supplier(java.util.function.Supplier) JobsLogs(io.crate.execution.engine.collect.stats.JobsLogs) ArrayList(java.util.ArrayList) BlockBasedRamAccounting(io.crate.breaker.BlockBasedRamAccounting) IntArrayList(com.carrotsearch.hppc.IntArrayList) Symbols(io.crate.expression.symbol.Symbols) Map(java.util.Map) BiConsumer(java.util.function.BiConsumer) CircuitBreaker(org.elasticsearch.common.breaker.CircuitBreaker) Nullable(javax.annotation.Nullable) LuceneReferenceResolver(io.crate.expression.reference.doc.lucene.LuceneReferenceResolver) Streamer(io.crate.Streamer) Collection(java.util.Collection) IndexService(org.elasticsearch.index.IndexService) Reference(io.crate.metadata.Reference) UUID(java.util.UUID) RamAccounting(io.crate.breaker.RamAccounting) LuceneCollectorExpression(io.crate.expression.reference.doc.lucene.LuceneCollectorExpression) TasksService(io.crate.execution.jobs.TasksService) IntObjectMap(com.carrotsearch.hppc.IntObjectMap) ConcurrentRamAccounting(io.crate.breaker.ConcurrentRamAccounting) IntObjectHashMap(com.carrotsearch.hppc.IntObjectHashMap) SQLExceptions(io.crate.exceptions.SQLExceptions) ThreadPools(io.crate.execution.support.ThreadPools) IntObjectHashMap(com.carrotsearch.hppc.IntObjectHashMap) ArrayList(java.util.ArrayList) IntArrayList(com.carrotsearch.hppc.IntArrayList) BlockBasedRamAccounting(io.crate.breaker.BlockBasedRamAccounting) RelationName(io.crate.metadata.RelationName) Supplier(java.util.function.Supplier) IntArrayList(com.carrotsearch.hppc.IntArrayList)

Example 7 with IntObjectMap

use of com.carrotsearch.hppc.IntObjectMap in project crate by crate.

the class NodeFetchOperation method fetch.

public CompletableFuture<? extends IntObjectMap<StreamBucket>> fetch(UUID jobId, int phaseId, @Nullable IntObjectMap<IntArrayList> docIdsToFetch, boolean closeTaskOnFinish) {
    if (docIdsToFetch == null) {
        if (closeTaskOnFinish) {
            tryCloseTask(jobId, phaseId);
        }
        jobsLogs.operationStarted(phaseId, jobId, "fetch", () -> -1);
        jobsLogs.operationFinished(phaseId, jobId, null);
        return CompletableFuture.completedFuture(new IntObjectHashMap<>(0));
    }
    RootTask context = tasksService.getTask(jobId);
    FetchTask fetchTask = context.getTask(phaseId);
    jobsLogs.operationStarted(phaseId, jobId, "fetch", () -> -1);
    BiConsumer<? super IntObjectMap<StreamBucket>, ? super Throwable> whenComplete = (res, err) -> {
        if (closeTaskOnFinish) {
            if (err == null) {
                fetchTask.close();
            } else {
                fetchTask.kill(err);
            }
        }
        if (err == null) {
            jobsLogs.operationFinished(phaseId, jobId, null);
        } else {
            jobsLogs.operationFinished(phaseId, jobId, SQLExceptions.messageOf(err));
        }
    };
    try {
        return doFetch(fetchTask, docIdsToFetch).whenComplete(whenComplete);
    } catch (Throwable t) {
        whenComplete.accept(null, t);
        return CompletableFuture.failedFuture(t);
    }
}
Also used : IntObjectCursor(com.carrotsearch.hppc.cursors.IntObjectCursor) StreamBucket(io.crate.execution.engine.distribution.StreamBucket) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) RelationName(io.crate.metadata.RelationName) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RootTask(io.crate.execution.jobs.RootTask) Supplier(java.util.function.Supplier) JobsLogs(io.crate.execution.engine.collect.stats.JobsLogs) ArrayList(java.util.ArrayList) BlockBasedRamAccounting(io.crate.breaker.BlockBasedRamAccounting) IntArrayList(com.carrotsearch.hppc.IntArrayList) Symbols(io.crate.expression.symbol.Symbols) Map(java.util.Map) BiConsumer(java.util.function.BiConsumer) CircuitBreaker(org.elasticsearch.common.breaker.CircuitBreaker) Nullable(javax.annotation.Nullable) LuceneReferenceResolver(io.crate.expression.reference.doc.lucene.LuceneReferenceResolver) Streamer(io.crate.Streamer) Collection(java.util.Collection) IndexService(org.elasticsearch.index.IndexService) Reference(io.crate.metadata.Reference) UUID(java.util.UUID) RamAccounting(io.crate.breaker.RamAccounting) LuceneCollectorExpression(io.crate.expression.reference.doc.lucene.LuceneCollectorExpression) TasksService(io.crate.execution.jobs.TasksService) IntObjectMap(com.carrotsearch.hppc.IntObjectMap) ConcurrentRamAccounting(io.crate.breaker.ConcurrentRamAccounting) IntObjectHashMap(com.carrotsearch.hppc.IntObjectHashMap) SQLExceptions(io.crate.exceptions.SQLExceptions) ThreadPools(io.crate.execution.support.ThreadPools) StreamBucket(io.crate.execution.engine.distribution.StreamBucket) RootTask(io.crate.execution.jobs.RootTask)

Aggregations

IntObjectMap (com.carrotsearch.hppc.IntObjectMap)7 IntObjectHashMap (com.carrotsearch.hppc.IntObjectHashMap)4 HashMap (java.util.HashMap)4 Map (java.util.Map)4 Streamer (io.crate.Streamer)3 RelationName (io.crate.metadata.RelationName)3 IntArrayList (com.carrotsearch.hppc.IntArrayList)2 IntSet (com.carrotsearch.hppc.IntSet)2 IntCursor (com.carrotsearch.hppc.cursors.IntCursor)2 IntObjectCursor (com.carrotsearch.hppc.cursors.IntObjectCursor)2 BlockBasedRamAccounting (io.crate.breaker.BlockBasedRamAccounting)2 ConcurrentRamAccounting (io.crate.breaker.ConcurrentRamAccounting)2 RamAccounting (io.crate.breaker.RamAccounting)2 SQLExceptions (io.crate.exceptions.SQLExceptions)2 JobsLogs (io.crate.execution.engine.collect.stats.JobsLogs)2 StreamBucket (io.crate.execution.engine.distribution.StreamBucket)2 RootTask (io.crate.execution.jobs.RootTask)2 TasksService (io.crate.execution.jobs.TasksService)2 ThreadPools (io.crate.execution.support.ThreadPools)2 LuceneCollectorExpression (io.crate.expression.reference.doc.lucene.LuceneCollectorExpression)2