Examples with AbstractUpdateRequest org.apache.solr.client.solrj.request.AbstractUpdateRequest used on opensource projects

Example 1 with AbstractUpdateRequest

use of org.apache.solr.client.solrj.request.AbstractUpdateRequest in project lucene-solr by apache.

the class RecoveryAfterSoftCommitTest method test.

@Test
public void test() throws Exception {
    waitForRecoveriesToFinish(DEFAULT_COLLECTION, true);
    // flush twice
    int i = 0;
    for (; i < MAX_BUFFERED_DOCS + 1; i++) {
        SolrInputDocument document = new SolrInputDocument();
        document.addField("id", String.valueOf(i));
        document.addField("a_t", "text_" + i);
        cloudClient.add(document);
    }
    // soft-commit so searchers are open on un-committed but flushed segment files
    AbstractUpdateRequest request = new UpdateRequest().setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true, true);
    cloudClient.request(request);
    Replica notLeader = ensureAllReplicasAreActive(DEFAULT_COLLECTION, "shard1", 1, 2, 30).get(0);
    // ok, now introduce a network partition between the leader and the replica
    SocketProxy proxy = getProxyForReplica(notLeader);
    proxy.close();
    // add more than ULOG_NUM_RECORDS_TO_KEEP docs so that peer sync cannot be used for recovery
    int MAX_DOCS = 2 + MAX_BUFFERED_DOCS + ULOG_NUM_RECORDS_TO_KEEP;
    for (; i < MAX_DOCS; i++) {
        SolrInputDocument document = new SolrInputDocument();
        document.addField("id", String.valueOf(i));
        document.addField("a_t", "text_" + i);
        cloudClient.add(document);
    }
    // Have the partition last at least 1 sec
    // While this gives the impression that recovery is timing related, this is
    // really only
    // to give time for the state to be written to ZK before the test completes.
    // In other words,
    // without a brief pause, the test finishes so quickly that it doesn't give
    // time for the recovery process to kick-in
    Thread.sleep(2000L);
    proxy.reopen();
    List<Replica> notLeaders = ensureAllReplicasAreActive(DEFAULT_COLLECTION, "shard1", 1, 2, 30);
}

Example 2 with AbstractUpdateRequest

use of org.apache.solr.client.solrj.request.AbstractUpdateRequest in project lucene-solr by apache.

the class CloudSolrClient method directUpdate.

private NamedList<Object> directUpdate(AbstractUpdateRequest request, String collection) throws SolrServerException {
    UpdateRequest updateRequest = (UpdateRequest) request;
    ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
    ModifiableSolrParams routableParams = new ModifiableSolrParams();
    ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
    if (params != null) {
        nonRoutableParams.add(params);
        routableParams.add(params);
        for (String param : NON_ROUTABLE_PARAMS) {
            routableParams.remove(param);
        }
    }
    if (collection == null) {
        throw new SolrServerException("No collection param specified on request and no default collection has been set.");
    }
    //Check to see if the collection is an alias.
    collection = stateProvider.getCollectionName(collection);
    DocCollection col = getDocCollection(collection, null);
    DocRouter router = col.getRouter();
    if (router instanceof ImplicitDocRouter) {
        // short circuit as optimization
        return null;
    }
    //Create the URL map, which is keyed on slice name.
    //The value is a list of URLs for each replica in the slice.
    //The first value in the list is the leader for the slice.
    final Map<String, List<String>> urlMap = buildUrlMap(col);
    final Map<String, LBHttpSolrClient.Req> routes = (urlMap == null ? null : updateRequest.getRoutes(router, col, urlMap, routableParams, this.idField));
    if (routes == null) {
        if (directUpdatesToLeadersOnly && hasInfoToFindLeaders(updateRequest, idField)) {
            // which to find the leaders but we could not find (all of) them
            throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "directUpdatesToLeadersOnly==true but could not find leader(s)");
        } else {
            // we could not find a leader or routes yet - use unoptimized general path
            return null;
        }
    }
    final NamedList<Throwable> exceptions = new NamedList<>();
    // +1 for deleteQuery
    final NamedList<NamedList> shardResponses = new NamedList<>(routes.size() + 1);
    long start = System.nanoTime();
    if (parallelUpdates) {
        final Map<String, Future<NamedList<?>>> responseFutures = new HashMap<>(routes.size());
        for (final Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) {
            final String url = entry.getKey();
            final LBHttpSolrClient.Req lbRequest = entry.getValue();
            try {
                MDC.put("CloudSolrClient.url", url);
                responseFutures.put(url, threadPool.submit(() -> lbClient.request(lbRequest).getResponse()));
            } finally {
                MDC.remove("CloudSolrClient.url");
            }
        }
        for (final Map.Entry<String, Future<NamedList<?>>> entry : responseFutures.entrySet()) {
            final String url = entry.getKey();
            final Future<NamedList<?>> responseFuture = entry.getValue();
            try {
                shardResponses.add(url, responseFuture.get());
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new RuntimeException(e);
            } catch (ExecutionException e) {
                exceptions.add(url, e.getCause());
            }
        }
        if (exceptions.size() > 0) {
            Throwable firstException = exceptions.getVal(0);
            if (firstException instanceof SolrException) {
                SolrException e = (SolrException) firstException;
                throw new RouteException(ErrorCode.getErrorCode(e.code()), exceptions, routes);
            } else {
                throw new RouteException(ErrorCode.SERVER_ERROR, exceptions, routes);
            }
        }
    } else {
        for (Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) {
            String url = entry.getKey();
            LBHttpSolrClient.Req lbRequest = entry.getValue();
            try {
                NamedList<Object> rsp = lbClient.request(lbRequest).getResponse();
                shardResponses.add(url, rsp);
            } catch (Exception e) {
                if (e instanceof SolrException) {
                    throw (SolrException) e;
                } else {
                    throw new SolrServerException(e);
                }
            }
        }
    }
    UpdateRequest nonRoutableRequest = null;
    List<String> deleteQuery = updateRequest.getDeleteQuery();
    if (deleteQuery != null && deleteQuery.size() > 0) {
        UpdateRequest deleteQueryRequest = new UpdateRequest();
        deleteQueryRequest.setDeleteQuery(deleteQuery);
        nonRoutableRequest = deleteQueryRequest;
    }
    Set<String> paramNames = nonRoutableParams.getParameterNames();
    Set<String> intersection = new HashSet<>(paramNames);
    intersection.retainAll(NON_ROUTABLE_PARAMS);
    if (nonRoutableRequest != null || intersection.size() > 0) {
        if (nonRoutableRequest == null) {
            nonRoutableRequest = new UpdateRequest();
        }
        nonRoutableRequest.setParams(nonRoutableParams);
        List<String> urlList = new ArrayList<>();
        urlList.addAll(routes.keySet());
        Collections.shuffle(urlList, rand);
        LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(nonRoutableRequest, urlList);
        try {
            LBHttpSolrClient.Rsp rsp = lbClient.request(req);
            shardResponses.add(urlList.get(0), rsp.getResponse());
        } catch (Exception e) {
            throw new SolrException(ErrorCode.SERVER_ERROR, urlList.get(0), e);
        }
    }
    long end = System.nanoTime();
    RouteResponse rr = condenseResponse(shardResponses, (int) TimeUnit.MILLISECONDS.convert(end - start, TimeUnit.NANOSECONDS));
    rr.setRouteResponses(shardResponses);
    rr.setRoutes(routes);
    return rr;
}

Example 3 with AbstractUpdateRequest

use of org.apache.solr.client.solrj.request.AbstractUpdateRequest in project lucene-solr by apache.

the class CloudSolrClient method sendRequest.

protected NamedList<Object> sendRequest(SolrRequest request, String collection) throws SolrServerException, IOException {
    connect();
    boolean sendToLeaders = false;
    List<String> replicas = null;
    if (request instanceof IsUpdateRequest) {
        if (request instanceof UpdateRequest) {
            NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, collection);
            if (response != null) {
                return response;
            }
        }
        sendToLeaders = true;
        replicas = new ArrayList<>();
    }
    SolrParams reqParams = request.getParams();
    if (reqParams == null) {
        reqParams = new ModifiableSolrParams();
    }
    List<String> theUrlList = new ArrayList<>();
    if (request instanceof V2Request) {
        Set<String> liveNodes = stateProvider.liveNodes();
        if (!liveNodes.isEmpty()) {
            List<String> liveNodesList = new ArrayList<>(liveNodes);
            Collections.shuffle(liveNodesList, rand);
            theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNodesList.get(0), (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME, "http")));
        }
    } else if (ADMIN_PATHS.contains(request.getPath())) {
        Set<String> liveNodes = stateProvider.liveNodes();
        for (String liveNode : liveNodes) {
            theUrlList.add(ZkStateReader.getBaseUrlForNodeName(liveNode, (String) stateProvider.getClusterProperty(ZkStateReader.URL_SCHEME, "http")));
        }
    } else {
        if (collection == null) {
            throw new SolrServerException("No collection param specified on request and no default collection has been set.");
        }
        Set<String> collectionNames = getCollectionNames(collection);
        if (collectionNames.size() == 0) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection: " + collection);
        }
        String shardKeys = reqParams.get(ShardParams._ROUTE_);
        // TODO: not a big deal because of the caching, but we could avoid looking
        // at every shard
        // when getting leaders if we tweaked some things
        // Retrieve slices from the cloud state and, for each collection
        // specified,
        // add it to the Map of slices.
        Map<String, Slice> slices = new HashMap<>();
        for (String collectionName : collectionNames) {
            DocCollection col = getDocCollection(collectionName, null);
            Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams, col);
            ClientUtils.addSlices(slices, collectionName, routeSlices, true);
        }
        Set<String> liveNodes = stateProvider.liveNodes();
        List<String> leaderUrlList = null;
        List<String> urlList = null;
        List<String> replicasList = null;
        // build a map of unique nodes
        // TODO: allow filtering by group, role, etc
        Map<String, ZkNodeProps> nodes = new HashMap<>();
        List<String> urlList2 = new ArrayList<>();
        for (Slice slice : slices.values()) {
            for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
                ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
                String node = coreNodeProps.getNodeName();
                if (!liveNodes.contains(coreNodeProps.getNodeName()) || Replica.State.getState(coreNodeProps.getState()) != Replica.State.ACTIVE)
                    continue;
                if (nodes.put(node, nodeProps) == null) {
                    if (!sendToLeaders || coreNodeProps.isLeader()) {
                        String url;
                        if (reqParams.get(UpdateParams.COLLECTION) == null) {
                            url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
                        } else {
                            url = coreNodeProps.getCoreUrl();
                        }
                        urlList2.add(url);
                    } else {
                        String url;
                        if (reqParams.get(UpdateParams.COLLECTION) == null) {
                            url = ZkCoreNodeProps.getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), collection);
                        } else {
                            url = coreNodeProps.getCoreUrl();
                        }
                        replicas.add(url);
                    }
                }
            }
        }
        if (sendToLeaders) {
            leaderUrlList = urlList2;
            replicasList = replicas;
        } else {
            urlList = urlList2;
        }
        if (sendToLeaders) {
            theUrlList = new ArrayList<>(leaderUrlList.size());
            theUrlList.addAll(leaderUrlList);
        } else {
            theUrlList = new ArrayList<>(urlList.size());
            theUrlList.addAll(urlList);
        }
        Collections.shuffle(theUrlList, rand);
        if (sendToLeaders) {
            ArrayList<String> theReplicas = new ArrayList<>(replicasList.size());
            theReplicas.addAll(replicasList);
            Collections.shuffle(theReplicas, rand);
            theUrlList.addAll(theReplicas);
        }
        if (theUrlList.isEmpty()) {
            for (String s : collectionNames) {
                if (s != null)
                    collectionStateCache.remove(s);
            }
            throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Could not find a healthy node to handle the request.");
        }
    }
    LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(request, theUrlList);
    LBHttpSolrClient.Rsp rsp = lbClient.request(req);
    return rsp.getResponse();
}

Example 4 with AbstractUpdateRequest

use of org.apache.solr.client.solrj.request.AbstractUpdateRequest in project lucene-solr by apache.

the class CloudSolrClientTest method testRouting.

@Test
public void testRouting() throws Exception {
    AbstractUpdateRequest request = new UpdateRequest().add(id, "0", "a_t", "hello1").add(id, "2", "a_t", "hello2").setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true);
    // Test single threaded routed updates for UpdateRequest
    NamedList<Object> response = getRandomClient().request(request, COLLECTION);
    if (getRandomClient().isDirectUpdatesToLeadersOnly()) {
        checkSingleServer(response);
    }
    CloudSolrClient.RouteResponse rr = (CloudSolrClient.RouteResponse) response;
    Map<String, LBHttpSolrClient.Req> routes = rr.getRoutes();
    Iterator<Map.Entry<String, LBHttpSolrClient.Req>> it = routes.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry<String, LBHttpSolrClient.Req> entry = it.next();
        String url = entry.getKey();
        UpdateRequest updateRequest = (UpdateRequest) entry.getValue().getRequest();
        SolrInputDocument doc = updateRequest.getDocuments().get(0);
        String id = doc.getField("id").getValue().toString();
        ModifiableSolrParams params = new ModifiableSolrParams();
        params.add("q", "id:" + id);
        params.add("distrib", "false");
        QueryRequest queryRequest = new QueryRequest(params);
        try (HttpSolrClient solrClient = getHttpSolrClient(url)) {
            QueryResponse queryResponse = queryRequest.process(solrClient);
            SolrDocumentList docList = queryResponse.getResults();
            assertTrue(docList.getNumFound() == 1);
        }
    }
    // Test the deleteById routing for UpdateRequest
    final UpdateResponse uResponse = new UpdateRequest().deleteById("0").deleteById("2").commit(cluster.getSolrClient(), COLLECTION);
    if (getRandomClient().isDirectUpdatesToLeadersOnly()) {
        checkSingleServer(uResponse.getResponse());
    }
    QueryResponse qResponse = getRandomClient().query(COLLECTION, new SolrQuery("*:*"));
    SolrDocumentList docs = qResponse.getResults();
    assertEquals(0, docs.getNumFound());
    // Test Multi-Threaded routed updates for UpdateRequest
    try (CloudSolrClient threadedClient = getCloudSolrClient(cluster.getZkServer().getZkAddress())) {
        threadedClient.setParallelUpdates(true);
        threadedClient.setDefaultCollection(COLLECTION);
        response = threadedClient.request(request);
        if (threadedClient.isDirectUpdatesToLeadersOnly()) {
            checkSingleServer(response);
        }
        rr = (CloudSolrClient.RouteResponse) response;
        routes = rr.getRoutes();
        it = routes.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, LBHttpSolrClient.Req> entry = it.next();
            String url = entry.getKey();
            UpdateRequest updateRequest = (UpdateRequest) entry.getValue().getRequest();
            SolrInputDocument doc = updateRequest.getDocuments().get(0);
            String id = doc.getField("id").getValue().toString();
            ModifiableSolrParams params = new ModifiableSolrParams();
            params.add("q", "id:" + id);
            params.add("distrib", "false");
            QueryRequest queryRequest = new QueryRequest(params);
            try (HttpSolrClient solrClient = getHttpSolrClient(url)) {
                QueryResponse queryResponse = queryRequest.process(solrClient);
                SolrDocumentList docList = queryResponse.getResults();
                assertTrue(docList.getNumFound() == 1);
            }
        }
    }
    // Test that queries with _route_ params are routed by the client
    // Track request counts on each node before query calls
    ClusterState clusterState = cluster.getSolrClient().getZkStateReader().getClusterState();
    DocCollection col = clusterState.getCollection(COLLECTION);
    Map<String, Long> requestCountsMap = Maps.newHashMap();
    for (Slice slice : col.getSlices()) {
        for (Replica replica : slice.getReplicas()) {
            String baseURL = (String) replica.get(ZkStateReader.BASE_URL_PROP);
            requestCountsMap.put(baseURL, getNumRequests(baseURL, COLLECTION));
        }
    }
    // Collect the base URLs of the replicas of shard that's expected to be hit
    DocRouter router = col.getRouter();
    Collection<Slice> expectedSlices = router.getSearchSlicesSingle("0", null, col);
    Set<String> expectedBaseURLs = Sets.newHashSet();
    for (Slice expectedSlice : expectedSlices) {
        for (Replica replica : expectedSlice.getReplicas()) {
            String baseURL = (String) replica.get(ZkStateReader.BASE_URL_PROP);
            expectedBaseURLs.add(baseURL);
        }
    }
    assertTrue("expected urls is not fewer than all urls! expected=" + expectedBaseURLs + "; all=" + requestCountsMap.keySet(), expectedBaseURLs.size() < requestCountsMap.size());
    // Calculate a number of shard keys that route to the same shard.
    int n;
    if (TEST_NIGHTLY) {
        n = random().nextInt(999) + 2;
    } else {
        n = random().nextInt(9) + 2;
    }
    List<String> sameShardRoutes = Lists.newArrayList();
    sameShardRoutes.add("0");
    for (int i = 1; i < n; i++) {
        String shardKey = Integer.toString(i);
        Collection<Slice> slices = router.getSearchSlicesSingle(shardKey, null, col);
        log.info("Expected Slices {}", slices);
        if (expectedSlices.equals(slices)) {
            sameShardRoutes.add(shardKey);
        }
    }
    assertTrue(sameShardRoutes.size() > 1);
    // Do N queries with _route_ parameter to the same shard
    for (int i = 0; i < n; i++) {
        ModifiableSolrParams solrParams = new ModifiableSolrParams();
        solrParams.set(CommonParams.Q, "*:*");
        solrParams.set(ShardParams._ROUTE_, sameShardRoutes.get(random().nextInt(sameShardRoutes.size())));
        log.info("output: {}", getRandomClient().query(COLLECTION, solrParams));
    }
    // Request counts increase from expected nodes should aggregate to 1000, while there should be
    // no increase in unexpected nodes.
    int increaseFromExpectedUrls = 0;
    int increaseFromUnexpectedUrls = 0;
    Map<String, Long> numRequestsToUnexpectedUrls = Maps.newHashMap();
    for (Slice slice : col.getSlices()) {
        for (Replica replica : slice.getReplicas()) {
            String baseURL = (String) replica.get(ZkStateReader.BASE_URL_PROP);
            Long prevNumRequests = requestCountsMap.get(baseURL);
            Long curNumRequests = getNumRequests(baseURL, COLLECTION);
            long delta = curNumRequests - prevNumRequests;
            if (expectedBaseURLs.contains(baseURL)) {
                increaseFromExpectedUrls += delta;
            } else {
                increaseFromUnexpectedUrls += delta;
                numRequestsToUnexpectedUrls.put(baseURL, delta);
            }
        }
    }
    assertEquals("Unexpected number of requests to expected URLs", n, increaseFromExpectedUrls);
    assertEquals("Unexpected number of requests to unexpected URLs: " + numRequestsToUnexpectedUrls, 0, increaseFromUnexpectedUrls);
}