Examples with ZkCoreNodeProps org.apache.solr.common.cloud.ZkCoreNodeProps used on opensource projects

Example 26 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class CloudSolrClient method buildUrlMap.

private Map<String, List<String>> buildUrlMap(DocCollection col) {
    Map<String, List<String>> urlMap = new HashMap<>();
    Collection<Slice> slices = col.getActiveSlices();
    Iterator<Slice> sliceIterator = slices.iterator();
    while (sliceIterator.hasNext()) {
        Slice slice = sliceIterator.next();
        String name = slice.getName();
        List<String> urls = new ArrayList<>();
        Replica leader = slice.getLeader();
        if (leader == null) {
            if (directUpdatesToLeadersOnly) {
                continue;
            }
            // take unoptimized general path - we cannot find a leader yet
            return null;
        }
        ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
        String url = zkProps.getCoreUrl();
        urls.add(url);
        if (!directUpdatesToLeadersOnly) {
            for (Replica replica : slice.getReplicas()) {
                if (!replica.getNodeName().equals(leader.getNodeName()) && !replica.getName().equals(leader.getName())) {
                    ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica);
                    String url1 = zkProps1.getCoreUrl();
                    urls.add(url1);
                }
            }
        }
        urlMap.put(name, urls);
    }
    return urlMap;
}

Example 27 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class CloudSolrClient method sendRequest.

protected NamedList<Object> sendRequest(SolrRequest request, String collection) throws SolrServerException, IOException {
    connect();
    boolean sendToLeaders = false;
    List<String> replicas = null;
    if (request instanceof IsUpdateRequest) {
        if (request instanceof UpdateRequest) {
            NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, collection);
            if (response != null) {
                return response;
            }
        }
        sendToLeaders = true;
        replicas = new ArrayList<>();
    }
    SolrParams reqParams = request.getParams();
    if (reqParams == null) {
        reqParams = new ModifiableSolrParams();
    }
    List<String> theUrlList = new ArrayList<>();
    if (request instanceof V2Request) {
        Set<String> liveNodes = stateProvider.liveNodes();
        if (!liveNodes.isEmpty()) {
            List<String> liveNodesList = new ArrayList<>(liveNodes);
            Collections.shuffle(liveNodesList, rand);
            theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNodesList.get(0), (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME, "http")));
        }
    } else if (ADMIN_PATHS.contains(request.getPath())) {
        Set<String> liveNodes = stateProvider.liveNodes();
        for (String liveNode : liveNodes) {
            theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode, (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME, "http")));
        }
    } else {
        if (collection == null) {
            throw new SolrServerException("No collection param specified on request and no default collection has been set.");
        }
        Set<String> collectionNames = getCollectionNames(collection);
        if (collectionNames.size() == 0) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection: " + collection);
        }
        String shardKeys = reqParams.get(ShardParams._ROUTE_);
        // TODO: not a big deal because of the caching, but we could avoid looking
        // at every shard
        // when getting leaders if we tweaked some things
        // Retrieve slices from the cloud state and, for each collection
        // specified,
        // add it to the Map of slices.
        Map<String, Slice> slices = new HashMap<>();
        for (String collectionName : collectionNames) {
            DocCollection col = getDocCollection(collectionName, null);
            Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams, col);
            ClientUtils.addSlices(slices, collectionName, routeSlices, true);
        }
        Set<String> liveNodes = stateProvider.liveNodes();
        List<String> leaderUrlList = null;
        List<String> urlList = null;
        List<String> replicasList = null;
        // build a map of unique nodes
        // TODO: allow filtering by group, role, etc
        Map<String, ZkNodeProps> nodes = new HashMap<>();
        List<String> urlList2 = new ArrayList<>();
        for (Slice slice : slices.values()) {
            for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
                ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
                String node = coreNodeProps.getNodeName();
                if (!liveNodes.contains(coreNodeProps.getNodeName()) || Replica.State.getState(coreNodeProps.getState()) != Replica.State.ACTIVE)
                    continue;
                if (nodes.put(node, nodeProps) == null) {
                    if (!sendToLeaders || coreNodeProps.isLeader()) {
                        String url;
                        if (reqParams.get(UpdateParams.COLLECTION) == null) {
                            url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
                        } else {
                            url = coreNodeProps.getCoreUrl();
                        }
                        urlList2.add(url);
                    } else {
                        String url;
                        if (reqParams.get(UpdateParams.COLLECTION) == null) {
                            url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
                        } else {
                            url = coreNodeProps.getCoreUrl();
                        }
                        replicas.add(url);
                    }
                }
            }
        }
        if (sendToLeaders) {
            leaderUrlList = urlList2;
            replicasList = replicas;
        } else {
            urlList = urlList2;
        }
        if (sendToLeaders) {
            theUrlList = new ArrayList<>(leaderUrlList.size());
            theUrlList.addAll(leaderUrlList);
        } else {
            theUrlList = new ArrayList<>(urlList.size());
            theUrlList.addAll(urlList);
        }
        Collections.shuffle(theUrlList, rand);
        if (sendToLeaders) {
            ArrayList<String> theReplicas = new ArrayList<>(replicasList.size());
            theReplicas.addAll(replicasList);
            Collections.shuffle(theReplicas, rand);
            theUrlList.addAll(theReplicas);
        }
        if (theUrlList.isEmpty()) {
            for (String s : collectionNames) {
                if (s != null)
                    collectionStateCache.remove(s);
            }
            throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Could not find a healthy node to handle the request.");
        }
    }
    LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(request, theUrlList);
    LBHttpSolrClient.Rsp rsp = lbClient.request(req);
    return rsp.getResponse();
}

Example 28 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class StatementImpl method constructStream.

protected SolrStream constructStream(String sql) throws IOException {
    try {
        ZkStateReader zkStateReader = this.connection.getClient().getZkStateReader();
        Collection<Slice> slices = CloudSolrStream.getSlices(this.connection.getCollection(), zkStateReader, true);
        List<Replica> shuffler = new ArrayList<>();
        for (Slice slice : slices) {
            Collection<Replica> replicas = slice.getReplicas();
            for (Replica replica : replicas) {
                shuffler.add(replica);
            }
        }
        Collections.shuffle(shuffler, new Random());
        ModifiableSolrParams params = new ModifiableSolrParams();
        params.set(CommonParams.QT, "/sql");
        params.set("stmt", sql);
        for (String propertyName : this.connection.getProperties().stringPropertyNames()) {
            params.set(propertyName, this.connection.getProperties().getProperty(propertyName));
        }
        Replica rep = shuffler.get(0);
        ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
        String url = zkProps.getCoreUrl();
        return new SolrStream(url, params);
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 29 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method reloadCollection.

protected boolean reloadCollection(Replica replica, String testCollectionName) throws Exception {
    ZkCoreNodeProps coreProps = new ZkCoreNodeProps(replica);
    String coreName = coreProps.getCoreName();
    boolean reloadedOk = false;
    try (HttpSolrClient client = getHttpSolrClient(coreProps.getBaseUrl())) {
        CoreAdminResponse statusResp = CoreAdminRequest.getStatus(coreName, client);
        long leaderCoreStartTime = statusResp.getStartTime(coreName).getTime();
        Thread.sleep(1000);
        // send reload command for the collection
        log.info("Sending RELOAD command for " + testCollectionName);
        ModifiableSolrParams params = new ModifiableSolrParams();
        params.set("action", CollectionParams.CollectionAction.RELOAD.toString());
        params.set("name", testCollectionName);
        QueryRequest request = new QueryRequest(params);
        request.setPath("/admin/collections");
        client.request(request);
        // reload can take a short while
        Thread.sleep(2000);
        // verify reload is done, waiting up to 30 seconds for slow test environments
        long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(30, TimeUnit.SECONDS);
        while (System.nanoTime() < timeout) {
            statusResp = CoreAdminRequest.getStatus(coreName, client);
            long startTimeAfterReload = statusResp.getStartTime(coreName).getTime();
            if (startTimeAfterReload > leaderCoreStartTime) {
                reloadedOk = true;
                break;
            }
            // else ... still waiting to see the reloaded core report a later start time
            Thread.sleep(1000);
        }
    }
    return reloadedOk;
}

Example 30 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method checkShardConsistency.

/* Checks shard consistency and optionally checks against the control shard.
   * The test will be failed if differences are found.
   */
protected void checkShardConsistency(boolean checkVsControl, boolean verbose, Set<String> addFails, Set<String> deleteFails) throws Exception {
    updateMappingsFromZk(jettys, clients, true);
    Set<String> theShards = shardToJetty.keySet();
    String failMessage = null;
    for (String shard : theShards) {
        String shardFailMessage = checkShardConsistency(shard, false, verbose);
        if (shardFailMessage != null && failMessage == null) {
            failMessage = shardFailMessage;
        }
    }
    if (failMessage != null) {
        fail(failMessage);
    }
    if (!checkVsControl)
        return;
    // add a tag to aid in debugging via logs
    SolrParams q = params("q", "*:*", "rows", "0", "tests", "checkShardConsistency(vsControl)");
    SolrDocumentList controlDocList = controlClient.query(q).getResults();
    long controlDocs = controlDocList.getNumFound();
    SolrDocumentList cloudDocList = cloudClient.query(q).getResults();
    long cloudClientDocs = cloudDocList.getNumFound();
    // now check that the right # are on each shard
    theShards = shardToJetty.keySet();
    int cnt = 0;
    for (String s : theShards) {
        int times = shardToJetty.get(s).size();
        for (int i = 0; i < times; i++) {
            try {
                CloudJettyRunner cjetty = shardToJetty.get(s).get(i);
                ZkNodeProps props = cjetty.info;
                SolrClient client = cjetty.client.solrClient;
                boolean active = Replica.State.getState(props.getStr(ZkStateReader.STATE_PROP)) == Replica.State.ACTIVE;
                if (active) {
                    SolrQuery query = new SolrQuery("*:*");
                    query.set("distrib", false);
                    long results = client.query(query).getResults().getNumFound();
                    if (verbose)
                        System.err.println(new ZkCoreNodeProps(props).getCoreUrl() + " : " + results);
                    if (verbose)
                        System.err.println("shard:" + props.getStr(ZkStateReader.SHARD_ID_PROP));
                    cnt += results;
                    break;
                }
            } catch (Exception e) {
                // if we have a problem, try the next one
                if (i == times - 1) {
                    throw e;
                }
            }
        }
    }
    if (controlDocs != cnt || cloudClientDocs != controlDocs) {
        String msg = "document count mismatch.  control=" + controlDocs + " sum(shards)=" + cnt + " cloudClient=" + cloudClientDocs;
        log.error(msg);
        boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient, addFails, deleteFails);
        if (shouldFail) {
            fail(msg);
        }
    }
}