Examples with V2Request org.apache.solr.client.solrj.request.V2Request used on opensource projects

Example 1 with V2Request

use of org.apache.solr.client.solrj.request.V2Request in project lucene-solr by apache.

the class CloudSolrClient method sendRequest.

protected NamedList<Object> sendRequest(SolrRequest request, String collection) throws SolrServerException, IOException {
    connect();
    boolean sendToLeaders = false;
    List<String> replicas = null;
    if (request instanceof IsUpdateRequest) {
        if (request instanceof UpdateRequest) {
            NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, collection);
            if (response != null) {
                return response;
            }
        }
        sendToLeaders = true;
        replicas = new ArrayList<>();
    }
    SolrParams reqParams = request.getParams();
    if (reqParams == null) {
        reqParams = new ModifiableSolrParams();
    }
    List<String> theUrlList = new ArrayList<>();
    if (request instanceof V2Request) {
        Set<String> liveNodes = stateProvider.liveNodes();
        if (!liveNodes.isEmpty()) {
            List<String> liveNodesList = new ArrayList<>(liveNodes);
            Collections.shuffle(liveNodesList, rand);
            theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNodesList.get(0), (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME, "http")));
        }
    } else if (ADMIN_PATHS.contains(request.getPath())) {
        Set<String> liveNodes = stateProvider.liveNodes();
        for (String liveNode : liveNodes) {
            theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode, (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME, "http")));
        }
    } else {
        if (collection == null) {
            throw new SolrServerException("No collection param specified on request and no default collection has been set.");
        }
        Set<String> collectionNames = getCollectionNames(collection);
        if (collectionNames.size() == 0) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection: " + collection);
        }
        String shardKeys = reqParams.get(ShardParams._ROUTE_);
        // TODO: not a big deal because of the caching, but we could avoid looking
        // at every shard
        // when getting leaders if we tweaked some things
        // Retrieve slices from the cloud state and, for each collection
        // specified,
        // add it to the Map of slices.
        Map<String, Slice> slices = new HashMap<>();
        for (String collectionName : collectionNames) {
            DocCollection col = getDocCollection(collectionName, null);
            Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams, col);
            ClientUtils.addSlices(slices, collectionName, routeSlices, true);
        }
        Set<String> liveNodes = stateProvider.liveNodes();
        List<String> leaderUrlList = null;
        List<String> urlList = null;
        List<String> replicasList = null;
        // build a map of unique nodes
        // TODO: allow filtering by group, role, etc
        Map<String, ZkNodeProps> nodes = new HashMap<>();
        List<String> urlList2 = new ArrayList<>();
        for (Slice slice : slices.values()) {
            for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
                ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
                String node = coreNodeProps.getNodeName();
                if (!liveNodes.contains(coreNodeProps.getNodeName()) || Replica.State.getState(coreNodeProps.getState()) != Replica.State.ACTIVE)
                    continue;
                if (nodes.put(node, nodeProps) == null) {
                    if (!sendToLeaders || coreNodeProps.isLeader()) {
                        String url;
                        if (reqParams.get(UpdateParams.COLLECTION) == null) {
                            url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
                        } else {
                            url = coreNodeProps.getCoreUrl();
                        }
                        urlList2.add(url);
                    } else {
                        String url;
                        if (reqParams.get(UpdateParams.COLLECTION) == null) {
                            url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
                        } else {
                            url = coreNodeProps.getCoreUrl();
                        }
                        replicas.add(url);
                    }
                }
            }
        }
        if (sendToLeaders) {
            leaderUrlList = urlList2;
            replicasList = replicas;
        } else {
            urlList = urlList2;
        }
        if (sendToLeaders) {
            theUrlList = new ArrayList<>(leaderUrlList.size());
            theUrlList.addAll(leaderUrlList);
        } else {
            theUrlList = new ArrayList<>(urlList.size());
            theUrlList.addAll(urlList);
        }
        Collections.shuffle(theUrlList, rand);
        if (sendToLeaders) {
            ArrayList<String> theReplicas = new ArrayList<>(replicasList.size());
            theReplicas.addAll(replicasList);
            Collections.shuffle(theReplicas, rand);
            theUrlList.addAll(theReplicas);
        }
        if (theUrlList.isEmpty()) {
            for (String s : collectionNames) {
                if (s != null)
                    collectionStateCache.remove(s);
            }
            throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Could not find a healthy node to handle the request.");
        }
    }
    LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(request, theUrlList);
    LBHttpSolrClient.Rsp rsp = lbClient.request(req);
    return rsp.getResponse();
}

Example 2 with V2Request

use of org.apache.solr.client.solrj.request.V2Request in project lucene-solr by apache.

the class CloudSolrClient method requestWithRetryOnStaleState.

/**
   * As this class doesn't watch external collections on the client side,
   * there's a chance that the request will fail due to cached stale state,
   * which means the state must be refreshed from ZK and retried.
   */
protected NamedList<Object> requestWithRetryOnStaleState(SolrRequest request, int retryCount, String collection) throws SolrServerException, IOException {
    // important to call this before you start working with the ZkStateReader
    connect();
    // build up a _stateVer_ param to pass to the server containing all of the
    // external collection state versions involved in this request, which allows
    // the server to notify us that our cached state for one or more of the external
    // collections is stale and needs to be refreshed ... this code has no impact on internal collections
    String stateVerParam = null;
    List<DocCollection> requestedCollections = null;
    boolean isCollectionRequestOfV2 = false;
    if (request instanceof V2Request) {
        isCollectionRequestOfV2 = ((V2Request) request).isPerCollectionRequest();
    }
    boolean isAdmin = ADMIN_PATHS.contains(request.getPath());
    if (collection != null && !isAdmin && !isCollectionRequestOfV2) {
        // don't do _stateVer_ checking for admin, v2 api requests
        Set<String> requestedCollectionNames = getCollectionNames(collection);
        StringBuilder stateVerParamBuilder = null;
        for (String requestedCollection : requestedCollectionNames) {
            // track the version of state we're using on the client side using the _stateVer_ param
            DocCollection coll = getDocCollection(requestedCollection, null);
            if (coll == null) {
                throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + requestedCollection);
            }
            int collVer = coll.getZNodeVersion();
            if (coll.getStateFormat() > 1) {
                if (requestedCollections == null)
                    requestedCollections = new ArrayList<>(requestedCollectionNames.size());
                requestedCollections.add(coll);
                if (stateVerParamBuilder == null) {
                    stateVerParamBuilder = new StringBuilder();
                } else {
                    // hopefully pipe is not an allowed char in a collection name
                    stateVerParamBuilder.append("|");
                }
                stateVerParamBuilder.append(coll.getName()).append(":").append(collVer);
            }
        }
        if (stateVerParamBuilder != null) {
            stateVerParam = stateVerParamBuilder.toString();
        }
    }
    if (request.getParams() instanceof ModifiableSolrParams) {
        ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
        if (stateVerParam != null) {
            params.set(STATE_VERSION, stateVerParam);
        } else {
            params.remove(STATE_VERSION);
        }
    }
    // else: ??? how to set this ???
    NamedList<Object> resp = null;
    try {
        resp = sendRequest(request, collection);
        //to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from there
        Object o = resp == null || resp.size() == 0 ? null : resp.get(STATE_VERSION, resp.size() - 1);
        if (o != null && o instanceof Map) {
            //remove this because no one else needs this and tests would fail if they are comparing responses
            resp.remove(resp.size() - 1);
            Map invalidStates = (Map) o;
            for (Object invalidEntries : invalidStates.entrySet()) {
                Map.Entry e = (Map.Entry) invalidEntries;
                getDocCollection((String) e.getKey(), (Integer) e.getValue());
            }
        }
    } catch (Exception exc) {
        Throwable rootCause = SolrException.getRootCause(exc);
        // or request is v2 api and its method is not GET
        if (collection == null || isAdmin || (request instanceof V2Request && request.getMethod() != SolrRequest.METHOD.GET)) {
            if (exc instanceof SolrServerException) {
                throw (SolrServerException) exc;
            } else if (exc instanceof IOException) {
                throw (IOException) exc;
            } else if (exc instanceof RuntimeException) {
                throw (RuntimeException) exc;
            } else {
                throw new SolrServerException(rootCause);
            }
        }
        int errorCode = (rootCause instanceof SolrException) ? ((SolrException) rootCause).code() : SolrException.ErrorCode.UNKNOWN.code;
        log.error("Request to collection {} failed due to (" + errorCode + ") {}, retry? " + retryCount, collection, rootCause.toString());
        boolean wasCommError = (rootCause instanceof ConnectException || rootCause instanceof ConnectTimeoutException || rootCause instanceof NoHttpResponseException || rootCause instanceof SocketException);
        if (wasCommError) {
            // in retryExpiryTime time
            for (DocCollection ext : requestedCollections) {
                ExpiringCachedDocCollection cacheEntry = collectionStateCache.get(ext.getName());
                if (cacheEntry == null)
                    continue;
                cacheEntry.maybeStale = true;
            }
            if (retryCount < MAX_STALE_RETRIES) {
                // the state would not have been updated
                return requestWithRetryOnStaleState(request, retryCount + 1, collection);
            }
        }
        boolean stateWasStale = false;
        if (retryCount < MAX_STALE_RETRIES && requestedCollections != null && !requestedCollections.isEmpty() && SolrException.ErrorCode.getErrorCode(errorCode) == SolrException.ErrorCode.INVALID_STATE) {
            // cached state for one or more external collections was stale
            // re-issue request using updated state
            stateWasStale = true;
            // just re-read state for all of them, which is a little heavy handed but hopefully a rare occurrence
            for (DocCollection ext : requestedCollections) {
                collectionStateCache.remove(ext.getName());
            }
        }
        // with ZK just to make sure the node we're trying to hit is still part of the collection
        if (retryCount < MAX_STALE_RETRIES && !stateWasStale && requestedCollections != null && !requestedCollections.isEmpty() && wasCommError) {
            for (DocCollection ext : requestedCollections) {
                DocCollection latestStateFromZk = getDocCollection(ext.getName(), null);
                if (latestStateFromZk.getZNodeVersion() != ext.getZNodeVersion()) {
                    // looks like we couldn't reach the server because the state was stale == retry
                    stateWasStale = true;
                    // we just pulled state from ZK, so update the cache so that the retry uses it
                    collectionStateCache.put(ext.getName(), new ExpiringCachedDocCollection(latestStateFromZk));
                }
            }
        }
        if (requestedCollections != null) {
            // done with this
            requestedCollections.clear();
        }
        // if the state was stale, then we retry the request once with new state pulled from Zk
        if (stateWasStale) {
            log.warn("Re-trying request to  collection(s) " + collection + " after stale state error from server.");
            resp = requestWithRetryOnStaleState(request, retryCount + 1, collection);
        } else {
            if (exc instanceof SolrException) {
                throw exc;
            }
            if (exc instanceof SolrServerException) {
                throw (SolrServerException) exc;
            } else if (exc instanceof IOException) {
                throw (IOException) exc;
            } else {
                throw new SolrServerException(rootCause);
            }
        }
    }
    return resp;
}

Example 3 with V2Request

use of org.apache.solr.client.solrj.request.V2Request in project lucene-solr by apache.

the class HttpSolrClient method createMethod.

protected HttpRequestBase createMethod(final SolrRequest request, String collection) throws IOException, SolrServerException {
    SolrParams params = request.getParams();
    Collection<ContentStream> streams = requestWriter.getContentStreams(request);
    String path = requestWriter.getPath(request);
    if (path == null || !path.startsWith("/")) {
        path = DEFAULT_PATH;
    }
    ResponseParser parser = request.getResponseParser();
    if (parser == null) {
        parser = this.parser;
    }
    // The parser 'wt=' and 'version=' params are used instead of the original
    // params
    ModifiableSolrParams wparams = new ModifiableSolrParams(params);
    if (parser != null) {
        wparams.set(CommonParams.WT, parser.getWriterType());
        wparams.set(CommonParams.VERSION, parser.getVersion());
    }
    if (invariantParams != null) {
        wparams.add(invariantParams);
    }
    String basePath = baseUrl;
    if (collection != null)
        basePath += "/" + collection;
    if (request instanceof V2Request) {
        if (System.getProperty("solr.v2RealPath") == null) {
            basePath = baseUrl.replace("/solr", "/v2");
        } else {
            basePath = baseUrl + "/____v2";
        }
    }
    if (SolrRequest.METHOD.GET == request.getMethod()) {
        if (streams != null) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
        }
        return new HttpGet(basePath + path + wparams.toQueryString());
    }
    if (SolrRequest.METHOD.DELETE == request.getMethod()) {
        return new HttpDelete(basePath + path + wparams.toQueryString());
    }
    if (SolrRequest.METHOD.POST == request.getMethod() || SolrRequest.METHOD.PUT == request.getMethod()) {
        String url = basePath + path;
        boolean hasNullStreamName = false;
        if (streams != null) {
            for (ContentStream cs : streams) {
                if (cs.getName() == null) {
                    hasNullStreamName = true;
                    break;
                }
            }
        }
        boolean isMultipart = ((this.useMultiPartPost && SolrRequest.METHOD.POST == request.getMethod()) || (streams != null && streams.size() > 1)) && !hasNullStreamName;
        LinkedList<NameValuePair> postOrPutParams = new LinkedList<>();
        if (streams == null || isMultipart) {
            // send server list and request list as query string params
            ModifiableSolrParams queryParams = calculateQueryParams(this.queryParams, wparams);
            queryParams.add(calculateQueryParams(request.getQueryParams(), wparams));
            String fullQueryUrl = url + queryParams.toQueryString();
            HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ? new HttpPost(fullQueryUrl) : new HttpPut(fullQueryUrl);
            if (!isMultipart) {
                postOrPut.addHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
            }
            List<FormBodyPart> parts = new LinkedList<>();
            Iterator<String> iter = wparams.getParameterNamesIterator();
            while (iter.hasNext()) {
                String p = iter.next();
                String[] vals = wparams.getParams(p);
                if (vals != null) {
                    for (String v : vals) {
                        if (isMultipart) {
                            parts.add(new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8)));
                        } else {
                            postOrPutParams.add(new BasicNameValuePair(p, v));
                        }
                    }
                }
            }
            // TODO: remove deprecated - first simple attempt failed, see {@link MultipartEntityBuilder}
            if (isMultipart && streams != null) {
                for (ContentStream content : streams) {
                    String contentType = content.getContentType();
                    if (contentType == null) {
                        // default
                        contentType = BinaryResponseParser.BINARY_CONTENT_TYPE;
                    }
                    String name = content.getName();
                    if (name == null) {
                        name = "";
                    }
                    parts.add(new FormBodyPart(name, new InputStreamBody(content.getStream(), contentType, content.getName())));
                }
            }
            if (parts.size() > 0) {
                MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT);
                for (FormBodyPart p : parts) {
                    entity.addPart(p);
                }
                postOrPut.setEntity(entity);
            } else {
                //not using multipart
                postOrPut.setEntity(new UrlEncodedFormEntity(postOrPutParams, StandardCharsets.UTF_8));
            }
            return postOrPut;
        } else // It is has one stream, it is the post body, put the params in the URL
        {
            String fullQueryUrl = url + wparams.toQueryString();
            HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ? new HttpPost(fullQueryUrl) : new HttpPut(fullQueryUrl);
            // Single stream as body
            // Using a loop just to get the first one
            final ContentStream[] contentStream = new ContentStream[1];
            for (ContentStream content : streams) {
                contentStream[0] = content;
                break;
            }
            if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
                Long size = contentStream[0].getSize();
                postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), size == null ? -1 : size) {

                    @Override
                    public Header getContentType() {
                        return new BasicHeader("Content-Type", contentStream[0].getContentType());
                    }

                    @Override
                    public boolean isRepeatable() {
                        return false;
                    }
                });
            } else {
                Long size = contentStream[0].getSize();
                postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), size == null ? -1 : size) {

                    @Override
                    public Header getContentType() {
                        return new BasicHeader("Content-Type", contentStream[0].getContentType());
                    }

                    @Override
                    public boolean isRepeatable() {
                        return false;
                    }
                });
            }
            return postOrPut;
        }
    }
    throw new SolrServerException("Unsupported method: " + request.getMethod());
}