Examples with ZkCoreNodeProps org.apache.solr.common.cloud.ZkCoreNodeProps used on opensource projects

Example 21 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class DistributedUpdateProcessor method getCollectionUrls.

private List<Node> getCollectionUrls(SolrQueryRequest req, String collection, EnumSet<Replica.Type> types) {
    ClusterState clusterState = req.getCore().getCoreContainer().getZkController().getClusterState();
    Map<String, Slice> slices = clusterState.getSlicesMap(collection);
    if (slices == null) {
        throw new ZooKeeperException(ErrorCode.BAD_REQUEST, "Could not find collection in zk: " + clusterState);
    }
    final List<Node> urls = new ArrayList<>(slices.size());
    for (Map.Entry<String, Slice> sliceEntry : slices.entrySet()) {
        Slice replicas = slices.get(sliceEntry.getKey());
        Map<String, Replica> shardMap = replicas.getReplicasMap();
        for (Entry<String, Replica> entry : shardMap.entrySet()) {
            if (!types.contains(entry.getValue().getType())) {
                continue;
            }
            ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(entry.getValue());
            if (clusterState.liveNodesContain(nodeProps.getNodeName())) {
                urls.add(new StdNode(nodeProps, collection, replicas.getName()));
            }
        }
    }
    if (urls.isEmpty()) {
        return null;
    }
    return urls;
}

Example 22 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class DistributedUpdateProcessor method getSubShardLeaders.

private List<Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc) {
    Collection<Slice> allSlices = coll.getSlices();
    List<Node> nodes = null;
    for (Slice aslice : allSlices) {
        final Slice.State state = aslice.getState();
        if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY) {
            DocRouter.Range myRange = coll.getSlice(shardId).getRange();
            if (myRange == null)
                myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
            boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
            if (isSubset && (// in case of deletes
            docId == null || (docId != null && coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll)))) {
                Replica sliceLeader = aslice.getLeader();
                // slice leader can be null because node/shard is created zk before leader election
                if (sliceLeader != null && zkController.getClusterState().liveNodesContain(sliceLeader.getNodeName())) {
                    if (nodes == null)
                        nodes = new ArrayList<>();
                    ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(sliceLeader);
                    nodes.add(new StdNode(nodeProps, coll.getName(), shardId));
                }
            }
        }
    }
    return nodes;
}

Example 23 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class DistributedUpdateProcessor method setupRequestForDBQ.

// used for deleteByQuery to get the list of nodes this leader should forward to
private List<Node> setupRequestForDBQ() {
    List<Node> nodes = null;
    String shardId = cloudDesc.getShardId();
    try {
        Replica leaderReplica = zkController.getZkStateReader().getLeaderRetry(collection, shardId);
        isLeader = leaderReplica.getName().equals(req.getCore().getCoreDescriptor().getCloudDescriptor().getCoreNodeName());
        // TODO: what if we are no longer the leader?
        forwardToLeader = false;
        List<ZkCoreNodeProps> replicaProps = zkController.getZkStateReader().getReplicaProps(collection, shardId, leaderReplica.getName(), null, Replica.State.DOWN, EnumSet.of(Replica.Type.NRT, Replica.Type.TLOG));
        if (replicaProps != null) {
            nodes = new ArrayList<>(replicaProps.size());
            for (ZkCoreNodeProps props : replicaProps) {
                nodes.add(new StdNode(props, collection, shardId));
            }
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
    }
    return nodes;
}

Example 24 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class DistributedUpdateProcessor method setupRequest.

private List<Node> setupRequest(String id, SolrInputDocument doc, String route) {
    List<Node> nodes = null;
    // if we are in zk mode...
    if (zkEnabled) {
        assert TestInjection.injectUpdateRandomPause();
        if ((updateCommand.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0) {
            // we actually might be the leader, but we don't want leader-logic for these types of updates anyway.
            isLeader = false;
            forwardToLeader = false;
            return nodes;
        }
        ClusterState cstate = zkController.getClusterState();
        DocCollection coll = cstate.getCollection(collection);
        Slice slice = coll.getRouter().getTargetSlice(id, doc, route, req.getParams(), coll);
        if (slice == null) {
            // No slice found.  Most strict routers will have already thrown an exception, so a null return is
            // a signal to use the slice of this core.
            // TODO: what if this core is not in the targeted collection?
            String shardId = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
            slice = coll.getSlice(shardId);
            if (slice == null) {
                throw new SolrException(ErrorCode.BAD_REQUEST, "No shard " + shardId + " in " + coll);
            }
        }
        DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
        if (DistribPhase.FROMLEADER == phase && !couldIbeSubShardLeader(coll)) {
            if (req.getCore().getCoreDescriptor().getCloudDescriptor().isLeader()) {
            // locally we think we are leader but the request says it came FROMLEADER
            // that could indicate a problem, let the full logic below figure it out
            } else {
                assert TestInjection.injectFailReplicaRequests();
                // we actually might be the leader, but we don't want leader-logic for these types of updates anyway.
                isLeader = false;
                forwardToLeader = false;
                return nodes;
            }
        }
        String shardId = slice.getName();
        try {
            // Not equivalent to getLeaderProps, which does retries to find a leader.
            // Replica leader = slice.getLeader();
            Replica leaderReplica = zkController.getZkStateReader().getLeaderRetry(collection, shardId);
            isLeader = leaderReplica.getName().equals(req.getCore().getCoreDescriptor().getCloudDescriptor().getCoreNodeName());
            if (!isLeader) {
                isSubShardLeader = amISubShardLeader(coll, slice, id, doc);
                if (isSubShardLeader) {
                    String myShardId = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
                    slice = coll.getSlice(myShardId);
                    shardId = myShardId;
                    leaderReplica = zkController.getZkStateReader().getLeaderRetry(collection, myShardId);
                    List<ZkCoreNodeProps> myReplicas = zkController.getZkStateReader().getReplicaProps(collection, shardId, leaderReplica.getName(), null, Replica.State.DOWN);
                }
            }
            doDefensiveChecks(phase);
            // if request is coming from another collection then we want it to be sent to all replicas
            // even if its phase is FROMLEADER
            String fromCollection = updateCommand.getReq().getParams().get(DISTRIB_FROM_COLLECTION);
            if (DistribPhase.FROMLEADER == phase && !isSubShardLeader && fromCollection == null) {
                // we are coming from the leader, just go local - add no urls
                forwardToLeader = false;
            } else if (isLeader || isSubShardLeader) {
                // that means I want to forward onto my replicas...
                // so get the replicas...
                forwardToLeader = false;
                List<ZkCoreNodeProps> replicaProps = zkController.getZkStateReader().getReplicaProps(collection, shardId, leaderReplica.getName(), null, Replica.State.DOWN);
                if (replicaProps != null) {
                    if (nodes == null) {
                        nodes = new ArrayList<>(replicaProps.size());
                    }
                    // check for test param that lets us miss replicas
                    String[] skipList = req.getParams().getParams(TEST_DISTRIB_SKIP_SERVERS);
                    Set<String> skipListSet = null;
                    if (skipList != null) {
                        skipListSet = new HashSet<>(skipList.length);
                        skipListSet.addAll(Arrays.asList(skipList));
                        log.info("test.distrib.skip.servers was found and contains:" + skipListSet);
                    }
                    for (ZkCoreNodeProps props : replicaProps) {
                        if (skipList != null) {
                            boolean skip = skipListSet.contains(props.getCoreUrl());
                            log.info("check url:" + props.getCoreUrl() + " against:" + skipListSet + " result:" + skip);
                            if (!skip) {
                                nodes.add(new StdNode(props, collection, shardId));
                            }
                        } else {
                            nodes.add(new StdNode(props, collection, shardId));
                        }
                    }
                }
            } else {
                // I need to forward onto the leader...
                nodes = new ArrayList<>(1);
                nodes.add(new RetryNode(new ZkCoreNodeProps(leaderReplica), zkController.getZkStateReader(), collection, shardId));
                forwardToLeader = true;
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
        }
    }
    return nodes;
}

Example 25 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class CloudSolrClient method getShardReplicationFactor.

/**
   * Walks the NamedList response after performing an update request looking for
   * the replication factor that was achieved in each shard involved in the request.
   * For single doc updates, there will be only one shard in the return value. 
   */
@SuppressWarnings("rawtypes")
public Map<String, Integer> getShardReplicationFactor(String collection, NamedList resp) {
    connect();
    Map<String, Integer> results = new HashMap<String, Integer>();
    if (resp instanceof CloudSolrClient.RouteResponse) {
        NamedList routes = ((CloudSolrClient.RouteResponse) resp).getRouteResponses();
        DocCollection coll = getDocCollection(collection, null);
        Map<String, String> leaders = new HashMap<String, String>();
        for (Slice slice : coll.getActiveSlices()) {
            Replica leader = slice.getLeader();
            if (leader != null) {
                ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
                String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName();
                leaders.put(leaderUrl, slice.getName());
                String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection;
                leaders.put(altLeaderUrl, slice.getName());
            }
        }
        Iterator<Map.Entry<String, Object>> routeIter = routes.iterator();
        while (routeIter.hasNext()) {
            Map.Entry<String, Object> next = routeIter.next();
            String host = next.getKey();
            NamedList hostResp = (NamedList) next.getValue();
            Integer rf = (Integer) ((NamedList) hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
            if (rf != null) {
                String shard = leaders.get(host);
                if (shard == null) {
                    if (host.endsWith("/"))
                        shard = leaders.get(host.substring(0, host.length() - 1));
                    if (shard == null) {
                        shard = host;
                    }
                }
                results.put(shard, rf);
            }
        }
    }
    return results;
}