Examples with IntCursor com.carrotsearch.hppc.cursors.IntCursor used on opensource projects

Example 11 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class ShardCollectSource method createMultiShardScoreDocCollector.

private CompletableFuture<BatchIterator<Row>> createMultiShardScoreDocCollector(RoutedCollectPhase collectPhase, boolean supportMoveToStart, CollectTask collectTask, String localNodeId) {
    Map<String, Map<String, IntIndexedContainer>> locations = collectPhase.routing().locations();
    SharedShardContexts sharedShardContexts = collectTask.sharedShardContexts();
    Map<String, IntIndexedContainer> indexShards = locations.get(localNodeId);
    List<CompletableFuture<OrderedDocCollector>> orderedDocCollectors = new ArrayList<>();
    Metadata metadata = clusterService.state().metadata();
    for (Map.Entry<String, IntIndexedContainer> entry : indexShards.entrySet()) {
        String indexName = entry.getKey();
        Index index = metadata.index(indexName).getIndex();
        for (IntCursor shard : entry.getValue()) {
            ShardId shardId = new ShardId(index, shard.value);
            try {
                SharedShardContext context = sharedShardContexts.getOrCreateContext(shardId);
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                orderedDocCollectors.add(shardCollectorProvider.getFutureOrderedCollector(collectPhase, context, collectTask, supportMoveToStart));
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                throw e;
            } catch (IndexNotFoundException e) {
                if (IndexParts.isPartitioned(indexName)) {
                    break;
                }
                throw e;
            }
        }
    }
    List<DataType<?>> columnTypes = Symbols.typeView(collectPhase.toCollect());
    OrderBy orderBy = collectPhase.orderBy();
    assert orderBy != null : "orderBy must not be null";
    return CompletableFutures.allAsList(orderedDocCollectors).thenApply(collectors -> OrderedLuceneBatchIteratorFactory.newInstance(collectors, OrderingByPosition.rowOrdering(OrderByPositionVisitor.orderByPositions(orderBy.orderBySymbols(), collectPhase.toCollect()), orderBy.reverseFlags(), orderBy.nullsFirst()), new RowAccountingWithEstimators(columnTypes, collectTask.getRamAccounting()), executor, availableThreads, supportMoveToStart));
}

Example 12 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class ShardCollectSource method getShardsIterator.

private Iterable<Row> getShardsIterator(TransactionContext txnCtx, RoutedCollectPhase collectPhase, String localNodeId) {
    Map<String, Map<String, IntIndexedContainer>> locations = collectPhase.routing().locations();
    List<UnassignedShard> unassignedShards = new ArrayList<>();
    List<ShardRowContext> shardRowContexts = new ArrayList<>();
    Map<String, IntIndexedContainer> indexShardsMap = locations.get(localNodeId);
    Metadata metadata = clusterService.state().metadata();
    for (Map.Entry<String, IntIndexedContainer> indexShards : indexShardsMap.entrySet()) {
        String indexName = indexShards.getKey();
        IndexMetadata indexMetadata = metadata.index(indexName);
        if (indexMetadata == null) {
            continue;
        }
        Index index = indexMetadata.getIndex();
        IntIndexedContainer shards = indexShards.getValue();
        IndexService indexService = indicesService.indexService(index);
        if (indexService == null) {
            for (IntCursor shard : shards) {
                unassignedShards.add(toUnassignedShard(index.getName(), UnassignedShard.markAssigned(shard.value)));
            }
            continue;
        }
        for (IntCursor shard : shards) {
            if (UnassignedShard.isUnassigned(shard.value)) {
                unassignedShards.add(toUnassignedShard(index.getName(), UnassignedShard.markAssigned(shard.value)));
                continue;
            }
            ShardId shardId = new ShardId(index, shard.value);
            try {
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                shardRowContexts.add(shardCollectorProvider.shardRowContext());
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                unassignedShards.add(toUnassignedShard(index.getName(), shard.value));
            }
        }
    }
    Iterable<Row> assignedShardRows = RowsTransformer.toRowsIterable(txnCtx, inputFactory, shardReferenceResolver, collectPhase, shardRowContexts, false);
    Iterable<Row> rows;
    if (unassignedShards.size() > 0) {
        Iterable<Row> unassignedShardRows = RowsTransformer.toRowsIterable(txnCtx, inputFactory, unassignedShardReferenceResolver, collectPhase, unassignedShards, false);
        rows = Iterables.concat(assignedShardRows, unassignedShardRows);
    } else {
        rows = assignedShardRows;
    }
    if (collectPhase.orderBy() != null) {
        return RowsTransformer.sortRows(Iterables.transform(rows, Row::materialize), collectPhase);
    }
    return rows;
}

Example 13 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class InternalCountOperation method count.

@Override
public CompletableFuture<Long> count(TransactionContext txnCtx, Map<String, IntIndexedContainer> indexShardMap, Symbol filter) {
    List<CompletableFuture<Supplier<Long>>> futureSuppliers = new ArrayList<>();
    Metadata metadata = clusterService.state().getMetadata();
    for (Map.Entry<String, IntIndexedContainer> entry : indexShardMap.entrySet()) {
        String indexName = entry.getKey();
        IndexMetadata indexMetadata = metadata.index(indexName);
        if (indexMetadata == null) {
            if (IndexParts.isPartitioned(indexName)) {
                continue;
            }
            throw new IndexNotFoundException(indexName);
        }
        final Index index = indexMetadata.getIndex();
        for (IntCursor shardCursor : entry.getValue()) {
            int shardValue = shardCursor.value;
            futureSuppliers.add(prepareGetCount(txnCtx, index, shardValue, filter));
        }
    }
    MergePartialCountFunction mergeFunction = new MergePartialCountFunction();
    return CompletableFutures.allAsList(futureSuppliers).thenCompose(suppliers -> ThreadPools.runWithAvailableThreads(executor, ThreadPools.numIdleThreads(executor, numProcessors), suppliers)).thenApply(mergeFunction);
}

Example 14 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class BulkShardResponseListener method toRowCounts.

/**
 * This calculates & returns the correct rowCount specific to the bulk individual bulk-params.
 *
 * Example:
 *
 * <pre>
 * where id = ? or id = ?
 * bulkParams: [ [1, 2], [3, 4] ]
 *
 * ->
 *  3 requests (per shard) (they can span across *all* parameters)
 *
 *   shard0: [0, 2]      // item locations
 *   shard1: [1]
 *   shard2: [3]
 *
 *  numBulkParams: 2 -> 2 items in the result
 *
 *  items:
 *     idx  value
 *      0 -> 0
 *      1 -> 0
 *      2 -> 1
 *      3 -> 1
 *
 *   result:
 *      long[] {2, 2}
 * </pre>
 */
private static long[] toRowCounts(ShardResponse.CompressedResult result, IntCollection items, int numBulkParams) {
    long[] rowCounts = new long[numBulkParams];
    Arrays.fill(rowCounts, 0L);
    for (IntCursor c : items) {
        int itemLocation = c.index;
        int resultIdx = c.value;
        if (result.successfulWrites(itemLocation)) {
            rowCounts[resultIdx]++;
        } else if (result.failed(itemLocation)) {
            rowCounts[resultIdx] = Row1.ERROR;
        }
    }
    return rowCounts;
}

Example 15 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class FetchProjection method generateStreamersGroupedByReaderAndNode.

@SuppressWarnings({ "rawtypes" })
public Map<String, ? extends IntObjectMap<Streamer[]>> generateStreamersGroupedByReaderAndNode() {
    HashMap<String, IntObjectHashMap<Streamer[]>> streamersByReaderByNode = new HashMap<>();
    for (Map.Entry<String, IntSet> entry : nodeReaders.entrySet()) {
        IntObjectHashMap<Streamer[]> streamersByReaderId = new IntObjectHashMap<>();
        String nodeId = entry.getKey();
        streamersByReaderByNode.put(nodeId, streamersByReaderId);
        for (IntCursor readerIdCursor : entry.getValue()) {
            int readerId = readerIdCursor.value;
            String index = readerIndices.floorEntry(readerId).getValue();
            RelationName relationName = indicesToIdents.get(index);
            FetchSource fetchSource = fetchSources.get(relationName);
            if (fetchSource == null) {
                continue;
            }
            streamersByReaderId.put(readerIdCursor.value, Symbols.streamerArray(fetchSource.references()));
        }
    }
    return streamersByReaderByNode;
}