Examples with DocCollection org.apache.solr.common.cloud.DocCollection used on opensource projects

Example 96 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class CloudSolrClient method getDocCollection.

protected DocCollection getDocCollection(String collection, Integer expectedVersion) throws SolrException {
    if (expectedVersion == null)
        expectedVersion = -1;
    if (collection == null)
        return null;
    ExpiringCachedDocCollection cacheEntry = collectionStateCache.get(collection);
    DocCollection col = cacheEntry == null ? null : cacheEntry.cached;
    if (col != null) {
        if (expectedVersion <= col.getZNodeVersion() && !cacheEntry.shoulRetry())
            return col;
    }
    ClusterState.CollectionRef ref = getCollectionRef(collection);
    if (ref == null) {
        //no such collection exists
        return null;
    }
    if (!ref.isLazilyLoaded()) {
        //it is readily available just return it
        return ref.get();
    }
    List locks = this.locks;
    final Object lock = locks.get(Math.abs(Hash.murmurhash3_x86_32(collection, 0, collection.length(), 0) % locks.size()));
    DocCollection fetchedCol = null;
    synchronized (lock) {
        /*we have waited for sometime just check once again*/
        cacheEntry = collectionStateCache.get(collection);
        col = cacheEntry == null ? null : cacheEntry.cached;
        if (col != null) {
            if (expectedVersion <= col.getZNodeVersion() && !cacheEntry.shoulRetry())
                return col;
        }
        // We are going to fetch a new version
        // we MUST try to get a new version
        //this is a call to ZK
        fetchedCol = ref.get();
        // this collection no more exists
        if (fetchedCol == null)
            return null;
        if (col != null && fetchedCol.getZNodeVersion() == col.getZNodeVersion()) {
            //we retried and found that it is the same version
            cacheEntry.setRetriedAt();
            cacheEntry.maybeStale = false;
        } else {
            if (fetchedCol.getStateFormat() > 1)
                collectionStateCache.put(collection, new ExpiringCachedDocCollection(fetchedCol));
        }
        return fetchedCol;
    }
}

Example 97 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class CloudSolrClient method directUpdate.

private NamedList<Object> directUpdate(AbstractUpdateRequest request, String collection) throws SolrServerException {
    UpdateRequest updateRequest = (UpdateRequest) request;
    ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
    ModifiableSolrParams routableParams = new ModifiableSolrParams();
    ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
    if (params != null) {
        nonRoutableParams.add(params);
        routableParams.add(params);
        for (String param : NON_ROUTABLE_PARAMS) {
            routableParams.remove(param);
        }
    }
    if (collection == null) {
        throw new SolrServerException("No collection param specified on request and no default collection has been set.");
    }
    //Check to see if the collection is an alias.
    collection = stateProvider.getCollectionName(collection);
    DocCollection col = getDocCollection(collection, null);
    DocRouter router = col.getRouter();
    if (router instanceof ImplicitDocRouter) {
        // short circuit as optimization
        return null;
    }
    //Create the URL map, which is keyed on slice name.
    //The value is a list of URLs for each replica in the slice.
    //The first value in the list is the leader for the slice.
    final Map<String, List<String>> urlMap = buildUrlMap(col);
    final Map<String, LBHttpSolrClient.Req> routes = (urlMap == null ? null : updateRequest.getRoutes(router, col, urlMap, routableParams, this.idField));
    if (routes == null) {
        if (directUpdatesToLeadersOnly && hasInfoToFindLeaders(updateRequest, idField)) {
            // which to find the leaders but we could not find (all of) them
            throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "directUpdatesToLeadersOnly==true but could not find leader(s)");
        } else {
            // we could not find a leader or routes yet - use unoptimized general path
            return null;
        }
    }
    final NamedList<Throwable> exceptions = new NamedList<>();
    // +1 for deleteQuery
    final NamedList<NamedList> shardResponses = new NamedList<>(routes.size() + 1);
    long start = System.nanoTime();
    if (parallelUpdates) {
        final Map<String, Future<NamedList<?>>> responseFutures = new HashMap<>(routes.size());
        for (final Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) {
            final String url = entry.getKey();
            final LBHttpSolrClient.Req lbRequest = entry.getValue();
            try {
                MDC.put("CloudSolrClient.url", url);
                responseFutures.put(url, threadPool.submit(() -> lbClient.request(lbRequest).getResponse()));
            } finally {
                MDC.remove("CloudSolrClient.url");
            }
        }
        for (final Map.Entry<String, Future<NamedList<?>>> entry : responseFutures.entrySet()) {
            final String url = entry.getKey();
            final Future<NamedList<?>> responseFuture = entry.getValue();
            try {
                shardResponses.add(url, responseFuture.get());
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new RuntimeException(e);
            } catch (ExecutionException e) {
                exceptions.add(url, e.getCause());
            }
        }
        if (exceptions.size() > 0) {
            Throwable firstException = exceptions.getVal(0);
            if (firstException instanceof SolrException) {
                SolrException e = (SolrException) firstException;
                throw new RouteException(ErrorCode.getErrorCode(e.code()), exceptions, routes);
            } else {
                throw new RouteException(ErrorCode.SERVER_ERROR, exceptions, routes);
            }
        }
    } else {
        for (Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) {
            String url = entry.getKey();
            LBHttpSolrClient.Req lbRequest = entry.getValue();
            try {
                NamedList<Object> rsp = lbClient.request(lbRequest).getResponse();
                shardResponses.add(url, rsp);
            } catch (Exception e) {
                if (e instanceof SolrException) {
                    throw (SolrException) e;
                } else {
                    throw new SolrServerException(e);
                }
            }
        }
    }
    UpdateRequest nonRoutableRequest = null;
    List<String> deleteQuery = updateRequest.getDeleteQuery();
    if (deleteQuery != null && deleteQuery.size() > 0) {
        UpdateRequest deleteQueryRequest = new UpdateRequest();
        deleteQueryRequest.setDeleteQuery(deleteQuery);
        nonRoutableRequest = deleteQueryRequest;
    }
    Set<String> paramNames = nonRoutableParams.getParameterNames();
    Set<String> intersection = new HashSet<>(paramNames);
    intersection.retainAll(NON_ROUTABLE_PARAMS);
    if (nonRoutableRequest != null || intersection.size() > 0) {
        if (nonRoutableRequest == null) {
            nonRoutableRequest = new UpdateRequest();
        }
        nonRoutableRequest.setParams(nonRoutableParams);
        List<String> urlList = new ArrayList<>();
        urlList.addAll(routes.keySet());
        Collections.shuffle(urlList, rand);
        LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(nonRoutableRequest, urlList);
        try {
            LBHttpSolrClient.Rsp rsp = lbClient.request(req);
            shardResponses.add(urlList.get(0), rsp.getResponse());
        } catch (Exception e) {
            throw new SolrException(ErrorCode.SERVER_ERROR, urlList.get(0), e);
        }
    }
    long end = System.nanoTime();
    RouteResponse rr = condenseResponse(shardResponses, (int) TimeUnit.MILLISECONDS.convert(end - start, TimeUnit.NANOSECONDS));
    rr.setRouteResponses(shardResponses);
    rr.setRoutes(routes);
    return rr;
}

Example 98 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class CloudSolrClient method getShardReplicationFactor.

/**
   * Walks the NamedList response after performing an update request looking for
   * the replication factor that was achieved in each shard involved in the request.
   * For single doc updates, there will be only one shard in the return value. 
   */
@SuppressWarnings("rawtypes")
public Map<String, Integer> getShardReplicationFactor(String collection, NamedList resp) {
    connect();
    Map<String, Integer> results = new HashMap<String, Integer>();
    if (resp instanceof CloudSolrClient.RouteResponse) {
        NamedList routes = ((CloudSolrClient.RouteResponse) resp).getRouteResponses();
        DocCollection coll = getDocCollection(collection, null);
        Map<String, String> leaders = new HashMap<String, String>();
        for (Slice slice : coll.getActiveSlices()) {
            Replica leader = slice.getLeader();
            if (leader != null) {
                ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
                String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName();
                leaders.put(leaderUrl, slice.getName());
                String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection;
                leaders.put(altLeaderUrl, slice.getName());
            }
        }
        Iterator<Map.Entry<String, Object>> routeIter = routes.iterator();
        while (routeIter.hasNext()) {
            Map.Entry<String, Object> next = routeIter.next();
            String host = next.getKey();
            NamedList hostResp = (NamedList) next.getValue();
            Integer rf = (Integer) ((NamedList) hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
            if (rf != null) {
                String shard = leaders.get(host);
                if (shard == null) {
                    if (host.endsWith("/"))
                        shard = leaders.get(host.substring(0, host.length() - 1));
                    if (shard == null) {
                        shard = host;
                    }
                }
                results.put(shard, rf);
            }
        }
    }
    return results;
}

Example 99 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class CloudSolrClient method sendRequest.

protected NamedList<Object> sendRequest(SolrRequest request, String collection) throws SolrServerException, IOException {
    connect();
    boolean sendToLeaders = false;
    List<String> replicas = null;
    if (request instanceof IsUpdateRequest) {
        if (request instanceof UpdateRequest) {
            NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, collection);
            if (response != null) {
                return response;
            }
        }
        sendToLeaders = true;
        replicas = new ArrayList<>();
    }
    SolrParams reqParams = request.getParams();
    if (reqParams == null) {
        reqParams = new ModifiableSolrParams();
    }
    List<String> theUrlList = new ArrayList<>();
    if (request instanceof V2Request) {
        Set<String> liveNodes = stateProvider.liveNodes();
        if (!liveNodes.isEmpty()) {
            List<String> liveNodesList = new ArrayList<>(liveNodes);
            Collections.shuffle(liveNodesList, rand);
            theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNodesList.get(0), (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME, "http")));
        }
    } else if (ADMIN_PATHS.contains(request.getPath())) {
        Set<String> liveNodes = stateProvider.liveNodes();
        for (String liveNode : liveNodes) {
            theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode, (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME, "http")));
        }
    } else {
        if (collection == null) {
            throw new SolrServerException("No collection param specified on request and no default collection has been set.");
        }
        Set<String> collectionNames = getCollectionNames(collection);
        if (collectionNames.size() == 0) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection: " + collection);
        }
        String shardKeys = reqParams.get(ShardParams._ROUTE_);
        // TODO: not a big deal because of the caching, but we could avoid looking
        // at every shard
        // when getting leaders if we tweaked some things
        // Retrieve slices from the cloud state and, for each collection
        // specified,
        // add it to the Map of slices.
        Map<String, Slice> slices = new HashMap<>();
        for (String collectionName : collectionNames) {
            DocCollection col = getDocCollection(collectionName, null);
            Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams, col);
            ClientUtils.addSlices(slices, collectionName, routeSlices, true);
        }
        Set<String> liveNodes = stateProvider.liveNodes();
        List<String> leaderUrlList = null;
        List<String> urlList = null;
        List<String> replicasList = null;
        // build a map of unique nodes
        // TODO: allow filtering by group, role, etc
        Map<String, ZkNodeProps> nodes = new HashMap<>();
        List<String> urlList2 = new ArrayList<>();
        for (Slice slice : slices.values()) {
            for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
                ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
                String node = coreNodeProps.getNodeName();
                if (!liveNodes.contains(coreNodeProps.getNodeName()) || Replica.State.getState(coreNodeProps.getState()) != Replica.State.ACTIVE)
                    continue;
                if (nodes.put(node, nodeProps) == null) {
                    if (!sendToLeaders || coreNodeProps.isLeader()) {
                        String url;
                        if (reqParams.get(UpdateParams.COLLECTION) == null) {
                            url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
                        } else {
                            url = coreNodeProps.getCoreUrl();
                        }
                        urlList2.add(url);
                    } else {
                        String url;
                        if (reqParams.get(UpdateParams.COLLECTION) == null) {
                            url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
                        } else {
                            url = coreNodeProps.getCoreUrl();
                        }
                        replicas.add(url);
                    }
                }
            }
        }
        if (sendToLeaders) {
            leaderUrlList = urlList2;
            replicasList = replicas;
        } else {
            urlList = urlList2;
        }
        if (sendToLeaders) {
            theUrlList = new ArrayList<>(leaderUrlList.size());
            theUrlList.addAll(leaderUrlList);
        } else {
            theUrlList = new ArrayList<>(urlList.size());
            theUrlList.addAll(urlList);
        }
        Collections.shuffle(theUrlList, rand);
        if (sendToLeaders) {
            ArrayList<String> theReplicas = new ArrayList<>(replicasList.size());
            theReplicas.addAll(replicasList);
            Collections.shuffle(theReplicas, rand);
            theUrlList.addAll(theReplicas);
        }
        if (theUrlList.isEmpty()) {
            for (String s : collectionNames) {
                if (s != null)
                    collectionStateCache.remove(s);
            }
            throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Could not find a healthy node to handle the request.");
        }
    }
    LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(request, theUrlList);
    LBHttpSolrClient.Rsp rsp = lbClient.request(req);
    return rsp.getResponse();
}

Example 100 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class CloudSolrClient method requestWithRetryOnStaleState.

/**
   * As this class doesn't watch external collections on the client side,
   * there's a chance that the request will fail due to cached stale state,
   * which means the state must be refreshed from ZK and retried.
   */
protected NamedList<Object> requestWithRetryOnStaleState(SolrRequest request, int retryCount, String collection) throws SolrServerException, IOException {
    // important to call this before you start working with the ZkStateReader
    connect();
    // build up a _stateVer_ param to pass to the server containing all of the
    // external collection state versions involved in this request, which allows
    // the server to notify us that our cached state for one or more of the external
    // collections is stale and needs to be refreshed ... this code has no impact on internal collections
    String stateVerParam = null;
    List<DocCollection> requestedCollections = null;
    boolean isCollectionRequestOfV2 = false;
    if (request instanceof V2Request) {
        isCollectionRequestOfV2 = ((V2Request) request).isPerCollectionRequest();
    }
    boolean isAdmin = ADMIN_PATHS.contains(request.getPath());
    if (collection != null && !isAdmin && !isCollectionRequestOfV2) {
        // don't do _stateVer_ checking for admin, v2 api requests
        Set<String> requestedCollectionNames = getCollectionNames(collection);
        StringBuilder stateVerParamBuilder = null;
        for (String requestedCollection : requestedCollectionNames) {
            // track the version of state we're using on the client side using the _stateVer_ param
            DocCollection coll = getDocCollection(requestedCollection, null);
            if (coll == null) {
                throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + requestedCollection);
            }
            int collVer = coll.getZNodeVersion();
            if (coll.getStateFormat() > 1) {
                if (requestedCollections == null)
                    requestedCollections = new ArrayList<>(requestedCollectionNames.size());
                requestedCollections.add(coll);
                if (stateVerParamBuilder == null) {
                    stateVerParamBuilder = new StringBuilder();
                } else {
                    // hopefully pipe is not an allowed char in a collection name
                    stateVerParamBuilder.append("|");
                }
                stateVerParamBuilder.append(coll.getName()).append(":").append(collVer);
            }
        }
        if (stateVerParamBuilder != null) {
            stateVerParam = stateVerParamBuilder.toString();
        }
    }
    if (request.getParams() instanceof ModifiableSolrParams) {
        ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
        if (stateVerParam != null) {
            params.set(STATE_VERSION, stateVerParam);
        } else {
            params.remove(STATE_VERSION);
        }
    }
    // else: ??? how to set this ???
    NamedList<Object> resp = null;
    try {
        resp = sendRequest(request, collection);
        //to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from there
        Object o = resp == null || resp.size() == 0 ? null : resp.get(STATE_VERSION, resp.size() - 1);
        if (o != null && o instanceof Map) {
            //remove this because no one else needs this and tests would fail if they are comparing responses
            resp.remove(resp.size() - 1);
            Map invalidStates = (Map) o;
            for (Object invalidEntries : invalidStates.entrySet()) {
                Map.Entry e = (Map.Entry) invalidEntries;
                getDocCollection((String) e.getKey(), (Integer) e.getValue());
            }
        }
    } catch (Exception exc) {
        Throwable rootCause = SolrException.getRootCause(exc);
        // or request is v2 api and its method is not GET
        if (collection == null || isAdmin || (request instanceof V2Request && request.getMethod() != SolrRequest.METHOD.GET)) {
            if (exc instanceof SolrServerException) {
                throw (SolrServerException) exc;
            } else if (exc instanceof IOException) {
                throw (IOException) exc;
            } else if (exc instanceof RuntimeException) {
                throw (RuntimeException) exc;
            } else {
                throw new SolrServerException(rootCause);
            }
        }
        int errorCode = (rootCause instanceof SolrException) ? ((SolrException) rootCause).code() : SolrException.ErrorCode.UNKNOWN.code;
        log.error("Request to collection {} failed due to (" + errorCode + ") {}, retry? " + retryCount, collection, rootCause.toString());
        boolean wasCommError = (rootCause instanceof ConnectException || rootCause instanceof ConnectTimeoutException || rootCause instanceof NoHttpResponseException || rootCause instanceof SocketException);
        if (wasCommError) {
            // in retryExpiryTime time
            for (DocCollection ext : requestedCollections) {
                ExpiringCachedDocCollection cacheEntry = collectionStateCache.get(ext.getName());
                if (cacheEntry == null)
                    continue;
                cacheEntry.maybeStale = true;
            }
            if (retryCount < MAX_STALE_RETRIES) {
                // the state would not have been updated
                return requestWithRetryOnStaleState(request, retryCount + 1, collection);
            }
        }
        boolean stateWasStale = false;
        if (retryCount < MAX_STALE_RETRIES && requestedCollections != null && !requestedCollections.isEmpty() && SolrException.ErrorCode.getErrorCode(errorCode) == SolrException.ErrorCode.INVALID_STATE) {
            // cached state for one or more external collections was stale
            // re-issue request using updated state
            stateWasStale = true;
            // just re-read state for all of them, which is a little heavy handed but hopefully a rare occurrence
            for (DocCollection ext : requestedCollections) {
                collectionStateCache.remove(ext.getName());
            }
        }
        // with ZK just to make sure the node we're trying to hit is still part of the collection
        if (retryCount < MAX_STALE_RETRIES && !stateWasStale && requestedCollections != null && !requestedCollections.isEmpty() && wasCommError) {
            for (DocCollection ext : requestedCollections) {
                DocCollection latestStateFromZk = getDocCollection(ext.getName(), null);
                if (latestStateFromZk.getZNodeVersion() != ext.getZNodeVersion()) {
                    // looks like we couldn't reach the server because the state was stale == retry
                    stateWasStale = true;
                    // we just pulled state from ZK, so update the cache so that the retry uses it
                    collectionStateCache.put(ext.getName(), new ExpiringCachedDocCollection(latestStateFromZk));
                }
            }
        }
        if (requestedCollections != null) {
            // done with this
            requestedCollections.clear();
        }
        // if the state was stale, then we retry the request once with new state pulled from Zk
        if (stateWasStale) {
            log.warn("Re-trying request to  collection(s) " + collection + " after stale state error from server.");
            resp = requestWithRetryOnStaleState(request, retryCount + 1, collection);
        } else {
            if (exc instanceof SolrException) {
                throw exc;
            }
            if (exc instanceof SolrServerException) {
                throw (SolrServerException) exc;
            } else if (exc instanceof IOException) {
                throw (IOException) exc;
            } else {
                throw new SolrServerException(rootCause);
            }
        }
    }
    return resp;
}