Examples with IntArrayList com.carrotsearch.hppc.IntArrayList used on opensource projects

Example 86 with IntArrayList

use of com.carrotsearch.hppc.IntArrayList in project crate by crate.

the class NodeFetchOperation method doFetch.

private CompletableFuture<? extends IntObjectMap<StreamBucket>> doFetch(FetchTask fetchTask, IntObjectMap<IntArrayList> toFetch) throws Exception {
    HashMap<RelationName, TableFetchInfo> tableFetchInfos = getTableFetchInfos(fetchTask);
    // RamAccounting is per doFetch call instead of per FetchTask/fetchPhase
    // To be able to free up the memory count when the operation is complete
    final var ramAccounting = ConcurrentRamAccounting.forCircuitBreaker("fetch-" + fetchTask.id(), circuitBreaker);
    ArrayList<Supplier<StreamBucket>> collectors = new ArrayList<>(toFetch.size());
    for (IntObjectCursor<IntArrayList> toFetchCursor : toFetch) {
        final int readerId = toFetchCursor.key;
        final IntArrayList docIds = toFetchCursor.value;
        RelationName ident = fetchTask.tableIdent(readerId);
        final TableFetchInfo tfi = tableFetchInfos.get(ident);
        assert tfi != null : "tfi must not be null";
        var collector = tfi.createCollector(readerId, new BlockBasedRamAccounting(ramAccounting::addBytes, BlockBasedRamAccounting.MAX_BLOCK_SIZE_IN_BYTES));
        collectors.add(() -> collector.collect(docIds));
    }
    return ThreadPools.runWithAvailableThreads(executor, ThreadPools.numIdleThreads(executor, numProcessors), collectors).thenApply(buckets -> {
        var toFetchIt = toFetch.iterator();
        assert toFetch.size() == buckets.size() : "Must have a bucket per reader and they must be in the same order";
        IntObjectHashMap<StreamBucket> bucketByReader = new IntObjectHashMap<>(toFetch.size());
        for (var bucket : buckets) {
            assert toFetchIt.hasNext() : "toFetchIt must have an element if there is one in buckets";
            int readerId = toFetchIt.next().key;
            bucketByReader.put(readerId, bucket);
        }
        return bucketByReader;
    }).whenComplete((result, err) -> ramAccounting.close());
}

Example 87 with IntArrayList

use of com.carrotsearch.hppc.IntArrayList in project crate by crate.

the class NodeFetchOperation method fetch.

public CompletableFuture<? extends IntObjectMap<StreamBucket>> fetch(UUID jobId, int phaseId, @Nullable IntObjectMap<IntArrayList> docIdsToFetch, boolean closeTaskOnFinish) {
    if (docIdsToFetch == null) {
        if (closeTaskOnFinish) {
            tryCloseTask(jobId, phaseId);
        }
        jobsLogs.operationStarted(phaseId, jobId, "fetch", () -> -1);
        jobsLogs.operationFinished(phaseId, jobId, null);
        return CompletableFuture.completedFuture(new IntObjectHashMap<>(0));
    }
    RootTask context = tasksService.getTask(jobId);
    FetchTask fetchTask = context.getTask(phaseId);
    jobsLogs.operationStarted(phaseId, jobId, "fetch", () -> -1);
    BiConsumer<? super IntObjectMap<StreamBucket>, ? super Throwable> whenComplete = (res, err) -> {
        if (closeTaskOnFinish) {
            if (err == null) {
                fetchTask.close();
            } else {
                fetchTask.kill(err);
            }
        }
        if (err == null) {
            jobsLogs.operationFinished(phaseId, jobId, null);
        } else {
            jobsLogs.operationFinished(phaseId, jobId, SQLExceptions.messageOf(err));
        }
    };
    try {
        return doFetch(fetchTask, docIdsToFetch).whenComplete(whenComplete);
    } catch (Throwable t) {
        whenComplete.accept(null, t);
        return CompletableFuture.failedFuture(t);
    }
}

Example 88 with IntArrayList

use of com.carrotsearch.hppc.IntArrayList in project crate by crate.

the class ReaderBucket method sortedDocs.

IntArrayList sortedDocs() {
    // are in the same order as the ids we requested.
    if (sortedDocs == null) {
        int[] keys = docs.keys().toArray();
        Arrays.sort(keys);
        sortedDocs = new IntArrayList(keys.length);
        sortedDocs.add(keys);
    }
    return sortedDocs;
}

Example 89 with IntArrayList

use of com.carrotsearch.hppc.IntArrayList in project crate by crate.

the class RoutingProvider method processShardRouting.

private static void processShardRouting(Map<String, Map<String, IntIndexedContainer>> locations, ShardRouting shardRouting) {
    String node = shardRouting.currentNodeId();
    Map<String, IntIndexedContainer> nodeMap = locations.get(node);
    if (nodeMap == null) {
        nodeMap = new TreeMap<>();
        locations.put(shardRouting.currentNodeId(), nodeMap);
    }
    String indexName = shardRouting.getIndexName();
    IntIndexedContainer shards = nodeMap.get(indexName);
    if (shards == null) {
        shards = new IntArrayList();
        nodeMap.put(indexName, shards);
    }
    shards.add(shardRouting.id());
}

Example 90 with IntArrayList

use of com.carrotsearch.hppc.IntArrayList in project crate by crate.

the class Union method pruneOutputsExcept.

@Override
public LogicalPlan pruneOutputsExcept(TableStats tableStats, Collection<Symbol> outputsToKeep) {
    IntArrayList outputIndicesToKeep = new IntArrayList();
    for (Symbol outputToKeep : outputsToKeep) {
        SymbolVisitors.intersection(outputToKeep, outputs, s -> {
            int idx = outputs.indexOf(s);
            assert idx >= 0 : "outputs must contain symbol " + s + " if intersection called consumer";
            outputIndicesToKeep.add(idx);
        });
    }
    ArrayList<Symbol> toKeepFromLhs = new ArrayList<>();
    ArrayList<Symbol> toKeepFromRhs = new ArrayList<>();
    ArrayList<Symbol> newOutputs = new ArrayList<>();
    for (IntCursor cursor : outputIndicesToKeep) {
        toKeepFromLhs.add(lhs.outputs().get(cursor.value));
        toKeepFromRhs.add(rhs.outputs().get(cursor.value));
        newOutputs.add(outputs.get(cursor.value));
    }
    LogicalPlan newLhs = lhs.pruneOutputsExcept(tableStats, toKeepFromLhs);
    LogicalPlan newRhs = rhs.pruneOutputsExcept(tableStats, toKeepFromRhs);
    if (newLhs == lhs && newRhs == rhs) {
        return this;
    }
    return new Union(newLhs, newRhs, newOutputs);
}