Examples with IntCursor com.carrotsearch.hppc.cursors.IntCursor used on opensource projects

Example 16 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class RoutingBuilder method allocateRoutingNodes.

private static void allocateRoutingNodes(Map<String, Map<Integer, String>> shardNodes, Map<String, Map<String, IntIndexedContainer>> locations) {
    for (Map.Entry<String, Map<String, IntIndexedContainer>> indicesByNodeId : locations.entrySet()) {
        String nodeId = indicesByNodeId.getKey();
        for (Map.Entry<String, IntIndexedContainer> shardsByIndexEntry : indicesByNodeId.getValue().entrySet()) {
            String index = shardsByIndexEntry.getKey();
            IntIndexedContainer shards = shardsByIndexEntry.getValue();
            Map<Integer, String> shardsOnIndex = shardNodes.get(index);
            if (shardsOnIndex == null) {
                shardsOnIndex = new HashMap<>(shards.size());
                shardNodes.put(index, shardsOnIndex);
                for (IntCursor id : shards) {
                    shardsOnIndex.put(id.value, nodeId);
                }
            } else {
                for (IntCursor id : shards) {
                    String allocatedNodeId = shardsOnIndex.get(id.value);
                    assert allocatedNodeId == null || allocatedNodeId.equals(nodeId) : "allocatedNodeId must match nodeId";
                    shardsOnIndex.put(id.value, nodeId);
                }
            }
        }
    }
}

Example 17 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class JobSetup method prepareOnRemote.

public List<CompletableFuture<StreamBucket>> prepareOnRemote(SessionSettings sessionInfo, Collection<? extends NodeOperation> nodeOperations, RootTask.Builder contextBuilder, SharedShardContexts sharedShardContexts) {
    Context context = new Context(clusterService.localNode().getId(), sessionInfo, contextBuilder, LOGGER, distributingConsumerFactory, nodeOperations, sharedShardContexts);
    registerContextPhases(nodeOperations, context);
    LOGGER.trace("prepareOnRemote: nodeOperations={}, targetSourceMap={}", nodeOperations, context.opCtx.targetToSourceMap);
    for (IntCursor cursor : context.opCtx.findLeafs()) {
        prepareSourceOperations(cursor.value, context);
    }
    assert context.opCtx.allNodeOperationContextsBuilt() : "some nodeOperations haven't been processed";
    return context.directResponseFutures;
}

Example 18 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class JobSetup method prepareSourceOperations.

/**
 * recursively build all contexts that depend on startPhaseId (excl. startPhaseId)
 * <p>
 * {@link Context#opCtx#targetToSourceMap} will be used to traverse the nodeOperations
 */
private void prepareSourceOperations(int startPhaseId, Context context) {
    IntContainer sourcePhaseIds = context.opCtx.targetToSourceMap.get(startPhaseId);
    if (sourcePhaseIds == null) {
        return;
    }
    for (IntCursor sourcePhaseId : sourcePhaseIds) {
        NodeOperation nodeOperation = context.opCtx.nodeOperationByPhaseId.get(sourcePhaseId.value);
        createContexts(nodeOperation.executionPhase(), context);
        context.opCtx.builtNodeOperations.set(nodeOperation.executionPhase().phaseId());
    }
    for (IntCursor sourcePhaseId : sourcePhaseIds) {
        prepareSourceOperations(sourcePhaseId.value, context);
    }
}

Example 19 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class FetchTask method start.

@Override
public void start() {
    synchronized (jobId) {
        if (killed != null) {
            result.completeExceptionally(killed);
            return;
        }
        HashMap<String, RelationName> index2TableIdent = new HashMap<>();
        for (Map.Entry<RelationName, Collection<String>> entry : phase.tableIndices().entrySet()) {
            for (String indexName : entry.getValue()) {
                index2TableIdent.put(indexName, entry.getKey());
            }
        }
        Set<RelationName> tablesWithFetchRefs = new HashSet<>();
        for (Reference reference : phase.fetchRefs()) {
            tablesWithFetchRefs.add(reference.ident().tableIdent());
        }
        String source = "fetch-task: " + jobId.toString() + '-' + phase.phaseId() + '-' + phase.name();
        for (Routing routing : routingIterable) {
            Map<String, Map<String, IntIndexedContainer>> locations = routing.locations();
            Map<String, IntIndexedContainer> indexShards = locations.get(localNodeId);
            for (Map.Entry<String, IntIndexedContainer> indexShardsEntry : indexShards.entrySet()) {
                String indexName = indexShardsEntry.getKey();
                Integer base = phase.bases().get(indexName);
                if (base == null) {
                    continue;
                }
                IndexMetadata indexMetadata = metadata.index(indexName);
                if (indexMetadata == null) {
                    if (IndexParts.isPartitioned(indexName)) {
                        continue;
                    }
                    throw new IndexNotFoundException(indexName);
                }
                Index index = indexMetadata.getIndex();
                RelationName ident = index2TableIdent.get(indexName);
                assert ident != null : "no relationName found for index " + indexName;
                tableIdents.put(base, ident);
                toFetch.put(ident, new ArrayList<>());
                for (IntCursor shard : indexShardsEntry.getValue()) {
                    ShardId shardId = new ShardId(index, shard.value);
                    int readerId = base + shardId.id();
                    SharedShardContext shardContext = shardContexts.get(readerId);
                    if (shardContext == null) {
                        try {
                            shardContext = sharedShardContexts.createContext(shardId, readerId);
                            shardContexts.put(readerId, shardContext);
                            if (tablesWithFetchRefs.contains(ident)) {
                                searchers.put(readerId, shardContext.acquireSearcher(source));
                            }
                        } catch (IndexNotFoundException e) {
                            if (!IndexParts.isPartitioned(indexName)) {
                                throw e;
                            }
                        }
                    }
                }
            }
        }
        for (Reference reference : phase.fetchRefs()) {
            Collection<Reference> references = toFetch.get(reference.ident().tableIdent());
            if (references != null) {
                references.add(reference);
            }
        }
    }
    if (searchers.isEmpty() || phase.fetchRefs().isEmpty()) {
        // no fetch references means there will be no fetch requests
        // this context is only here to allow the collectors to generate docids with the right bases
        // the bases are fetched in the prepare phase therefore this context can be closed
        close();
    }
}

Example 20 with IntCursor

use of com.carrotsearch.hppc.cursors.IntCursor in project crate by crate.

the class Routing method writeTo.

@Override
public void writeTo(StreamOutput out) throws IOException {
    out.writeVInt(locations.size());
    for (Map.Entry<String, Map<String, IntIndexedContainer>> entry : locations.entrySet()) {
        out.writeString(entry.getKey());
        Map<String, IntIndexedContainer> shardsByIndex = entry.getValue();
        if (shardsByIndex == null) {
            out.writeVInt(0);
        } else {
            out.writeVInt(shardsByIndex.size());
            for (Map.Entry<String, IntIndexedContainer> innerEntry : shardsByIndex.entrySet()) {
                out.writeString(innerEntry.getKey());
                IntIndexedContainer shardIds = innerEntry.getValue();
                if (shardIds == null || shardIds.size() == 0) {
                    out.writeVInt(0);
                } else {
                    out.writeVInt(shardIds.size());
                    for (IntCursor shardId : shardIds) {
                        out.writeVInt(shardId.value);
                    }
                }
            }
        }
    }
}