Examples with IntIndexedContainer com.carrotsearch.hppc.IntIndexedContainer used on opensource projects

Example 6 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class RoutingBuilder method allocateRoutingNodes.

private static void allocateRoutingNodes(Map<String, Map<Integer, String>> shardNodes, Map<String, Map<String, IntIndexedContainer>> locations) {
    for (Map.Entry<String, Map<String, IntIndexedContainer>> indicesByNodeId : locations.entrySet()) {
        String nodeId = indicesByNodeId.getKey();
        for (Map.Entry<String, IntIndexedContainer> shardsByIndexEntry : indicesByNodeId.getValue().entrySet()) {
            String index = shardsByIndexEntry.getKey();
            IntIndexedContainer shards = shardsByIndexEntry.getValue();
            Map<Integer, String> shardsOnIndex = shardNodes.get(index);
            if (shardsOnIndex == null) {
                shardsOnIndex = new HashMap<>(shards.size());
                shardNodes.put(index, shardsOnIndex);
                for (IntCursor id : shards) {
                    shardsOnIndex.put(id.value, nodeId);
                }
            } else {
                for (IntCursor id : shards) {
                    String allocatedNodeId = shardsOnIndex.get(id.value);
                    assert allocatedNodeId == null || allocatedNodeId.equals(nodeId) : "allocatedNodeId must match nodeId";
                    shardsOnIndex.put(id.value, nodeId);
                }
            }
        }
    }
}

Example 7 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class SysShardsTableInfo method getRouting.

/**
 * Retrieves the routing for sys.shards
 * <p>
 * This routing contains ALL shards of ALL indices.
 * Any shards that are not yet assigned to a node will have a NEGATIVE shard id (see {@link UnassignedShard}
 */
public static Routing getRouting(ClusterState clusterState, RoutingProvider routingProvider, SessionContext sessionContext) {
    String[] concreteIndices = Arrays.stream(clusterState.metadata().getConcreteAllIndices()).filter(index -> !IndexParts.isDangling(index)).toArray(String[]::new);
    User user = sessionContext != null ? sessionContext.sessionUser() : null;
    if (user != null) {
        List<String> accessibleTables = new ArrayList<>(concreteIndices.length);
        for (String indexName : concreteIndices) {
            String tableName = RelationName.fqnFromIndexName(indexName);
            if (user.hasAnyPrivilege(Privilege.Clazz.TABLE, tableName)) {
                accessibleTables.add(indexName);
            }
        }
        concreteIndices = accessibleTables.toArray(new String[0]);
    }
    Map<String, Map<String, IntIndexedContainer>> locations = new TreeMap<>();
    GroupShardsIterator<ShardIterator> groupShardsIterator = clusterState.getRoutingTable().allAssignedShardsGrouped(concreteIndices, true);
    for (final ShardIterator shardIt : groupShardsIterator) {
        final ShardRouting shardRouting = shardIt.nextOrNull();
        processShardRouting(clusterState.getNodes().getLocalNodeId(), locations, shardRouting, shardIt.shardId());
    }
    return new Routing(locations);
}

Example 8 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class SysShardsTableInfo method processShardRouting.

private static void processShardRouting(String localNodeId, Map<String, Map<String, IntIndexedContainer>> routing, ShardRouting shardRouting, ShardId shardId) {
    String node;
    int id;
    String index = shardId.getIndex().getName();
    if (shardRouting == null) {
        node = localNodeId;
        id = UnassignedShard.markUnassigned(shardId.id());
    } else {
        node = shardRouting.currentNodeId();
        id = shardRouting.id();
    }
    Map<String, IntIndexedContainer> nodeMap = routing.computeIfAbsent(node, k -> new TreeMap<>());
    IntIndexedContainer shards = nodeMap.get(index);
    if (shards == null) {
        shards = new IntArrayList();
        nodeMap.put(index, shards);
    }
    shards.add(id);
}

Example 9 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class FetchTask method start.

@Override
public void start() {
    synchronized (jobId) {
        if (killed != null) {
            result.completeExceptionally(killed);
            return;
        }
        HashMap<String, RelationName> index2TableIdent = new HashMap<>();
        for (Map.Entry<RelationName, Collection<String>> entry : phase.tableIndices().entrySet()) {
            for (String indexName : entry.getValue()) {
                index2TableIdent.put(indexName, entry.getKey());
            }
        }
        Set<RelationName> tablesWithFetchRefs = new HashSet<>();
        for (Reference reference : phase.fetchRefs()) {
            tablesWithFetchRefs.add(reference.ident().tableIdent());
        }
        String source = "fetch-task: " + jobId.toString() + '-' + phase.phaseId() + '-' + phase.name();
        for (Routing routing : routingIterable) {
            Map<String, Map<String, IntIndexedContainer>> locations = routing.locations();
            Map<String, IntIndexedContainer> indexShards = locations.get(localNodeId);
            for (Map.Entry<String, IntIndexedContainer> indexShardsEntry : indexShards.entrySet()) {
                String indexName = indexShardsEntry.getKey();
                Integer base = phase.bases().get(indexName);
                if (base == null) {
                    continue;
                }
                IndexMetadata indexMetadata = metadata.index(indexName);
                if (indexMetadata == null) {
                    if (IndexParts.isPartitioned(indexName)) {
                        continue;
                    }
                    throw new IndexNotFoundException(indexName);
                }
                Index index = indexMetadata.getIndex();
                RelationName ident = index2TableIdent.get(indexName);
                assert ident != null : "no relationName found for index " + indexName;
                tableIdents.put(base, ident);
                toFetch.put(ident, new ArrayList<>());
                for (IntCursor shard : indexShardsEntry.getValue()) {
                    ShardId shardId = new ShardId(index, shard.value);
                    int readerId = base + shardId.id();
                    SharedShardContext shardContext = shardContexts.get(readerId);
                    if (shardContext == null) {
                        try {
                            shardContext = sharedShardContexts.createContext(shardId, readerId);
                            shardContexts.put(readerId, shardContext);
                            if (tablesWithFetchRefs.contains(ident)) {
                                searchers.put(readerId, shardContext.acquireSearcher(source));
                            }
                        } catch (IndexNotFoundException e) {
                            if (!IndexParts.isPartitioned(indexName)) {
                                throw e;
                            }
                        }
                    }
                }
            }
        }
        for (Reference reference : phase.fetchRefs()) {
            Collection<Reference> references = toFetch.get(reference.ident().tableIdent());
            if (references != null) {
                references.add(reference);
            }
        }
    }
    if (searchers.isEmpty() || phase.fetchRefs().isEmpty()) {
        // no fetch references means there will be no fetch requests
        // this context is only here to allow the collectors to generate docids with the right bases
        // the bases are fetched in the prepare phase therefore this context can be closed
        close();
    }
}

Example 10 with IntIndexedContainer

use of com.carrotsearch.hppc.IntIndexedContainer in project crate by crate.

the class Routing method forTableOnAllNodes.

public static Routing forTableOnAllNodes(RelationName relationName, DiscoveryNodes nodes) {
    TreeMap<String, Map<String, IntIndexedContainer>> indicesByNode = new TreeMap<>();
    Map<String, IntIndexedContainer> shardsByIndex = Collections.singletonMap(relationName.indexNameOrAlias(), IntArrayList.from(IntArrayList.EMPTY_ARRAY));
    for (DiscoveryNode node : nodes) {
        indicesByNode.put(node.getId(), shardsByIndex);
    }
    return new Routing(indicesByNode);
}