Examples with IntIndexedContainer com.carrotsearch.hppc.IntIndexedContainer used on opensource projects

Example 1 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project graphhopper by graphhopper.

the class PrepareRoutingSubnetworks method keepLargeNetworks.

/**
 * Deletes all but the largest subnetworks.
 */
int keepLargeNetworks(PrepEdgeFilter filter, List<IntArrayList> components) {
    if (components.size() <= 1)
        return 0;
    int maxCount = -1;
    IntIndexedContainer oldComponent = null;
    int allRemoved = 0;
    FlagEncoder encoder = filter.getEncoder();
    EdgeExplorer explorer = ghStorage.createEdgeExplorer(filter);
    for (IntArrayList component : components) {
        if (maxCount < 0) {
            maxCount = component.size();
            oldComponent = component;
            continue;
        }
        int removedEdges;
        if (maxCount < component.size()) {
            // new biggest area found. remove old
            removedEdges = removeEdges(explorer, encoder, oldComponent, minNetworkSize);
            maxCount = component.size();
            oldComponent = component;
        } else {
            removedEdges = removeEdges(explorer, encoder, component, minNetworkSize);
        }
        allRemoved += removedEdges;
    }
    if (allRemoved > ghStorage.getAllEdges().getMaxId() / 2)
        throw new IllegalStateException("Too many total edges were removed: " + allRemoved + ", all edges:" + ghStorage.getAllEdges().getMaxId());
    return allRemoved;
}

Example 2 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class ShardCollectSource method createMultiShardScoreDocCollector.

private CompletableFuture<BatchIterator<Row>> createMultiShardScoreDocCollector(RoutedCollectPhase collectPhase, boolean supportMoveToStart, CollectTask collectTask, String localNodeId) {
    Map<String, Map<String, IntIndexedContainer>> locations = collectPhase.routing().locations();
    SharedShardContexts sharedShardContexts = collectTask.sharedShardContexts();
    Map<String, IntIndexedContainer> indexShards = locations.get(localNodeId);
    List<CompletableFuture<OrderedDocCollector>> orderedDocCollectors = new ArrayList<>();
    Metadata metadata = clusterService.state().metadata();
    for (Map.Entry<String, IntIndexedContainer> entry : indexShards.entrySet()) {
        String indexName = entry.getKey();
        Index index = metadata.index(indexName).getIndex();
        for (IntCursor shard : entry.getValue()) {
            ShardId shardId = new ShardId(index, shard.value);
            try {
                SharedShardContext context = sharedShardContexts.getOrCreateContext(shardId);
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                orderedDocCollectors.add(shardCollectorProvider.getFutureOrderedCollector(collectPhase, context, collectTask, supportMoveToStart));
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                throw e;
            } catch (IndexNotFoundException e) {
                if (IndexParts.isPartitioned(indexName)) {
                    break;
                }
                throw e;
            }
        }
    }
    List<DataType<?>> columnTypes = Symbols.typeView(collectPhase.toCollect());
    OrderBy orderBy = collectPhase.orderBy();
    assert orderBy != null : "orderBy must not be null";
    return CompletableFutures.allAsList(orderedDocCollectors).thenApply(collectors -> OrderedLuceneBatchIteratorFactory.newInstance(collectors, OrderingByPosition.rowOrdering(OrderByPositionVisitor.orderByPositions(orderBy.orderBySymbols(), collectPhase.toCollect()), orderBy.reverseFlags(), orderBy.nullsFirst()), new RowAccountingWithEstimators(columnTypes, collectTask.getRamAccounting()), executor, availableThreads, supportMoveToStart));
}

Example 3 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class ShardCollectSource method getShardsIterator.

private Iterable<Row> getShardsIterator(TransactionContext txnCtx, RoutedCollectPhase collectPhase, String localNodeId) {
    Map<String, Map<String, IntIndexedContainer>> locations = collectPhase.routing().locations();
    List<UnassignedShard> unassignedShards = new ArrayList<>();
    List<ShardRowContext> shardRowContexts = new ArrayList<>();
    Map<String, IntIndexedContainer> indexShardsMap = locations.get(localNodeId);
    Metadata metadata = clusterService.state().metadata();
    for (Map.Entry<String, IntIndexedContainer> indexShards : indexShardsMap.entrySet()) {
        String indexName = indexShards.getKey();
        IndexMetadata indexMetadata = metadata.index(indexName);
        if (indexMetadata == null) {
            continue;
        }
        Index index = indexMetadata.getIndex();
        IntIndexedContainer shards = indexShards.getValue();
        IndexService indexService = indicesService.indexService(index);
        if (indexService == null) {
            for (IntCursor shard : shards) {
                unassignedShards.add(toUnassignedShard(index.getName(), UnassignedShard.markAssigned(shard.value)));
            }
            continue;
        }
        for (IntCursor shard : shards) {
            if (UnassignedShard.isUnassigned(shard.value)) {
                unassignedShards.add(toUnassignedShard(index.getName(), UnassignedShard.markAssigned(shard.value)));
                continue;
            }
            ShardId shardId = new ShardId(index, shard.value);
            try {
                ShardCollectorProvider shardCollectorProvider = getCollectorProviderSafe(shardId);
                shardRowContexts.add(shardCollectorProvider.shardRowContext());
            } catch (ShardNotFoundException | IllegalIndexShardStateException e) {
                unassignedShards.add(toUnassignedShard(index.getName(), shard.value));
            }
        }
    }
    Iterable<Row> assignedShardRows = RowsTransformer.toRowsIterable(txnCtx, inputFactory, shardReferenceResolver, collectPhase, shardRowContexts, false);
    Iterable<Row> rows;
    if (unassignedShards.size() > 0) {
        Iterable<Row> unassignedShardRows = RowsTransformer.toRowsIterable(txnCtx, inputFactory, unassignedShardReferenceResolver, collectPhase, unassignedShards, false);
        rows = Iterables.concat(assignedShardRows, unassignedShardRows);
    } else {
        rows = assignedShardRows;
    }
    if (collectPhase.orderBy() != null) {
        return RowsTransformer.sortRows(Iterables.transform(rows, Row::materialize), collectPhase);
    }
    return rows;
}

Example 4 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class InternalCountOperation method count.

@Override
public CompletableFuture<Long> count(TransactionContext txnCtx, Map<String, IntIndexedContainer> indexShardMap, Symbol filter) {
    List<CompletableFuture<Supplier<Long>>> futureSuppliers = new ArrayList<>();
    Metadata metadata = clusterService.state().getMetadata();
    for (Map.Entry<String, IntIndexedContainer> entry : indexShardMap.entrySet()) {
        String indexName = entry.getKey();
        IndexMetadata indexMetadata = metadata.index(indexName);
        if (indexMetadata == null) {
            if (IndexParts.isPartitioned(indexName)) {
                continue;
            }
            throw new IndexNotFoundException(indexName);
        }
        final Index index = indexMetadata.getIndex();
        for (IntCursor shardCursor : entry.getValue()) {
            int shardValue = shardCursor.value;
            futureSuppliers.add(prepareGetCount(txnCtx, index, shardValue, filter));
        }
    }
    MergePartialCountFunction mergeFunction = new MergePartialCountFunction();
    return CompletableFutures.allAsList(futureSuppliers).thenCompose(suppliers -> ThreadPools.runWithAvailableThreads(executor, ThreadPools.numIdleThreads(executor, numProcessors), suppliers)).thenApply(mergeFunction);
}

Example 5 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class RoutingBuilder method buildReaderAllocations.

ReaderAllocations buildReaderAllocations() {
    Map<RelationName, Collection<String>> indicesByTable = new HashMap<>();
    IndexBaseBuilder indexBaseBuilder = new IndexBaseBuilder();
    Map<String, Map<Integer, String>> shardNodes = new HashMap<>();
    Map<RelationName, List<Routing>> routingListByTable = routingListByTableStack.removeLast();
    assert routingListByTable != null : "Call to `buildReaderAllocations` without prior `newAllocations` call";
    for (var tableRouting : routingListByTable.entrySet()) {
        RelationName table = tableRouting.getKey();
        List<Routing> routingList = tableRouting.getValue();
        for (Routing routing : routingList) {
            allocateRoutingNodes(shardNodes, routing.locations());
            for (Map.Entry<String, Map<String, IntIndexedContainer>> entry : routing.locations().entrySet()) {
                Map<String, IntIndexedContainer> shardsByIndex = entry.getValue();
                Collection<String> indices = indicesByTable.computeIfAbsent(table, ignored -> new ArrayList<>());
                indices.addAll(shardsByIndex.keySet());
                for (Map.Entry<String, IntIndexedContainer> shardsByIndexEntry : shardsByIndex.entrySet()) {
                    indexBaseBuilder.allocate(shardsByIndexEntry.getKey(), shardsByIndexEntry.getValue());
                }
            }
        }
    }
    return new ReaderAllocations(indexBaseBuilder.build(), shardNodes, indicesByTable);
}