Examples with Node org.apache.solr.update.SolrCmdDistributor.Node used on opensource projects

Example 6 with Node

use of org.apache.solr.update.SolrCmdDistributor.Node in project lucene-solr by apache.

the class DistributedUpdateProcessor method processCommit.

@Override
public void processCommit(CommitUpdateCommand cmd) throws IOException {
    assert TestInjection.injectFailUpdateRequests();
    updateCommand = cmd;
    List<Node> nodes = null;
    boolean singleLeader = false;
    if (zkEnabled) {
        zkCheck();
        nodes = getCollectionUrls(req, req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(), EnumSet.of(Replica.Type.TLOG, Replica.Type.NRT));
        if (nodes == null) {
            // This could happen if there are only pull replicas
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to distribute commit operation. No replicas available of types " + Replica.Type.TLOG + " or " + Replica.Type.NRT);
        }
        if (isLeader && nodes.size() == 1 && replicaType != Replica.Type.PULL) {
            singleLeader = true;
        }
    }
    if (!zkEnabled || req.getParams().getBool(COMMIT_END_POINT, false) || singleLeader) {
        if (replicaType == Replica.Type.TLOG) {
            try {
                Replica leaderReplica = zkController.getZkStateReader().getLeaderRetry(collection, cloudDesc.getShardId());
                isLeader = leaderReplica.getName().equals(req.getCore().getCoreDescriptor().getCloudDescriptor().getCoreNodeName());
                if (isLeader) {
                    long commitVersion = vinfo.getNewClock();
                    cmd.setVersion(commitVersion);
                    doLocalCommit(cmd);
                } else {
                    assert TestInjection.waitForInSyncWithLeader(req.getCore(), zkController, collection, cloudDesc.getShardId()) : "Core " + req.getCore() + " not in sync with leader";
                }
            } catch (InterruptedException e) {
                throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Exception finding leader for shard " + cloudDesc.getShardId(), e);
            }
        } else if (replicaType == Replica.Type.PULL) {
            log.warn("Commit not supported on replicas of type " + Replica.Type.PULL);
        } else {
            // NRT replicas will always commit
            if (vinfo != null) {
                long commitVersion = vinfo.getNewClock();
                cmd.setVersion(commitVersion);
            }
            doLocalCommit(cmd);
        }
    } else {
        ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
        if (!req.getParams().getBool(COMMIT_END_POINT, false)) {
            params.set(COMMIT_END_POINT, true);
            params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString());
            params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName()));
            if (nodes != null) {
                cmdDistrib.distribCommit(cmd, nodes, params);
                cmdDistrib.blockAndDoRetries();
            }
        }
    }
}

Example 7 with Node

use of org.apache.solr.update.SolrCmdDistributor.Node in project lucene-solr by apache.

the class DistributedUpdateProcessor method setupRequest.

private List<Node> setupRequest(String id, SolrInputDocument doc, String route) {
    List<Node> nodes = null;
    // if we are in zk mode...
    if (zkEnabled) {
        assert TestInjection.injectUpdateRandomPause();
        if ((updateCommand.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0) {
            // we actually might be the leader, but we don't want leader-logic for these types of updates anyway.
            isLeader = false;
            forwardToLeader = false;
            return nodes;
        }
        ClusterState cstate = zkController.getClusterState();
        DocCollection coll = cstate.getCollection(collection);
        Slice slice = coll.getRouter().getTargetSlice(id, doc, route, req.getParams(), coll);
        if (slice == null) {
            // No slice found.  Most strict routers will have already thrown an exception, so a null return is
            // a signal to use the slice of this core.
            // TODO: what if this core is not in the targeted collection?
            String shardId = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
            slice = coll.getSlice(shardId);
            if (slice == null) {
                throw new SolrException(ErrorCode.BAD_REQUEST, "No shard " + shardId + " in " + coll);
            }
        }
        DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
        if (DistribPhase.FROMLEADER == phase && !couldIbeSubShardLeader(coll)) {
            if (req.getCore().getCoreDescriptor().getCloudDescriptor().isLeader()) {
            // locally we think we are leader but the request says it came FROMLEADER
            // that could indicate a problem, let the full logic below figure it out
            } else {
                assert TestInjection.injectFailReplicaRequests();
                // we actually might be the leader, but we don't want leader-logic for these types of updates anyway.
                isLeader = false;
                forwardToLeader = false;
                return nodes;
            }
        }
        String shardId = slice.getName();
        try {
            // Not equivalent to getLeaderProps, which does retries to find a leader.
            // Replica leader = slice.getLeader();
            Replica leaderReplica = zkController.getZkStateReader().getLeaderRetry(collection, shardId);
            isLeader = leaderReplica.getName().equals(req.getCore().getCoreDescriptor().getCloudDescriptor().getCoreNodeName());
            if (!isLeader) {
                isSubShardLeader = amISubShardLeader(coll, slice, id, doc);
                if (isSubShardLeader) {
                    String myShardId = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
                    slice = coll.getSlice(myShardId);
                    shardId = myShardId;
                    leaderReplica = zkController.getZkStateReader().getLeaderRetry(collection, myShardId);
                    List<ZkCoreNodeProps> myReplicas = zkController.getZkStateReader().getReplicaProps(collection, shardId, leaderReplica.getName(), null, Replica.State.DOWN);
                }
            }
            doDefensiveChecks(phase);
            // if request is coming from another collection then we want it to be sent to all replicas
            // even if its phase is FROMLEADER
            String fromCollection = updateCommand.getReq().getParams().get(DISTRIB_FROM_COLLECTION);
            if (DistribPhase.FROMLEADER == phase && !isSubShardLeader && fromCollection == null) {
                // we are coming from the leader, just go local - add no urls
                forwardToLeader = false;
            } else if (isLeader || isSubShardLeader) {
                // that means I want to forward onto my replicas...
                // so get the replicas...
                forwardToLeader = false;
                List<ZkCoreNodeProps> replicaProps = zkController.getZkStateReader().getReplicaProps(collection, shardId, leaderReplica.getName(), null, Replica.State.DOWN);
                if (replicaProps != null) {
                    if (nodes == null) {
                        nodes = new ArrayList<>(replicaProps.size());
                    }
                    // check for test param that lets us miss replicas
                    String[] skipList = req.getParams().getParams(TEST_DISTRIB_SKIP_SERVERS);
                    Set<String> skipListSet = null;
                    if (skipList != null) {
                        skipListSet = new HashSet<>(skipList.length);
                        skipListSet.addAll(Arrays.asList(skipList));
                        log.info("test.distrib.skip.servers was found and contains:" + skipListSet);
                    }
                    for (ZkCoreNodeProps props : replicaProps) {
                        if (skipList != null) {
                            boolean skip = skipListSet.contains(props.getCoreUrl());
                            log.info("check url:" + props.getCoreUrl() + " against:" + skipListSet + " result:" + skip);
                            if (!skip) {
                                nodes.add(new StdNode(props, collection, shardId));
                            }
                        } else {
                            nodes.add(new StdNode(props, collection, shardId));
                        }
                    }
                }
            } else {
                // I need to forward onto the leader...
                nodes = new ArrayList<>(1);
                nodes.add(new RetryNode(new ZkCoreNodeProps(leaderReplica), zkController.getZkStateReader(), collection, shardId));
                forwardToLeader = true;
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
        }
    }
    return nodes;
}

Example 8 with Node

use of org.apache.solr.update.SolrCmdDistributor.Node in project lucene-solr by apache.

the class SolrCmdDistributorTest method testRetryNodeWontRetrySocketError.

private void testRetryNodeWontRetrySocketError() throws Exception {
    final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
    long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults().getNumFound();
    final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
    SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0);
    streamingClients.setExp(Exp.SOCKET_EXCEPTION);
    ArrayList<Node> nodes = new ArrayList<>();
    ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
    final AtomicInteger retries = new AtomicInteger();
    nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
    RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {

        @Override
        public boolean checkRetry() {
            retries.incrementAndGet();
            return true;
        }
    };
    nodes.add(retryNode);
    AddUpdateCommand cmd = new AddUpdateCommand(null);
    cmd.solrDoc = sdoc("id", id.incrementAndGet());
    ModifiableSolrParams params = new ModifiableSolrParams();
    CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
    cmdDistrib.distribAdd(cmd, nodes, params);
    streamingClients.setExp(null);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    // it will checkRetry, but not actually do it...
    assertEquals(1, retries.get());
    long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults().getNumFound();
    // we will get java.net.SocketException: Network is unreachable, which we don't retry on
    assertEquals(numFoundBefore, numFoundAfter);
    assertEquals(1, cmdDistrib.getErrors().size());
}

Example 9 with Node

use of org.apache.solr.update.SolrCmdDistributor.Node in project lucene-solr by apache.

the class SolrCmdDistributorTest method test.

@Test
@ShardsFixed(num = 4)
public void test() throws Exception {
    del("*:*");
    SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    ModifiableSolrParams params = new ModifiableSolrParams();
    List<Node> nodes = new ArrayList<>();
    ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, ((HttpSolrClient) controlClient).getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
    nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps)));
    // add one doc to controlClient
    AddUpdateCommand cmd = new AddUpdateCommand(null);
    cmd.solrDoc = sdoc("id", id.incrementAndGet());
    params = new ModifiableSolrParams();
    cmdDistrib.distribAdd(cmd, nodes, params);
    CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    List<Error> errors = cmdDistrib.getErrors();
    assertEquals(errors.toString(), 0, errors.size());
    long numFound = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
    assertEquals(1, numFound);
    HttpSolrClient client = (HttpSolrClient) clients.get(0);
    nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, client.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
    nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps)));
    // add another 2 docs to control and 3 to client
    cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    cmd.solrDoc = sdoc("id", id.incrementAndGet());
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribAdd(cmd, nodes, params);
    int id2 = id.incrementAndGet();
    AddUpdateCommand cmd2 = new AddUpdateCommand(null);
    cmd2.solrDoc = sdoc("id", id2);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribAdd(cmd2, nodes, params);
    AddUpdateCommand cmd3 = new AddUpdateCommand(null);
    cmd3.solrDoc = sdoc("id", id.incrementAndGet());
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribAdd(cmd3, Collections.singletonList(nodes.get(1)), params);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    errors = cmdDistrib.getErrors();
    assertEquals(errors.toString(), 0, errors.size());
    SolrDocumentList results = controlClient.query(new SolrQuery("*:*")).getResults();
    numFound = results.getNumFound();
    assertEquals(results.toString(), 3, numFound);
    numFound = client.query(new SolrQuery("*:*")).getResults().getNumFound();
    assertEquals(3, numFound);
    // now delete doc 2 which is on both control and client1
    DeleteUpdateCommand dcmd = new DeleteUpdateCommand(null);
    dcmd.id = Integer.toString(id2);
    cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribDelete(dcmd, nodes, params);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    errors = cmdDistrib.getErrors();
    assertEquals(errors.toString(), 0, errors.size());
    results = controlClient.query(new SolrQuery("*:*")).getResults();
    numFound = results.getNumFound();
    assertEquals(results.toString(), 2, numFound);
    numFound = client.query(new SolrQuery("*:*")).getResults().getNumFound();
    assertEquals(results.toString(), 2, numFound);
    for (SolrClient c : clients) {
        c.optimize();
    //System.out.println(clients.get(0).request(new LukeRequest()));
    }
    cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    int cnt = atLeast(303);
    for (int i = 0; i < cnt; i++) {
        nodes.clear();
        for (SolrClient c : clients) {
            if (random().nextBoolean()) {
                continue;
            }
            HttpSolrClient httpClient = (HttpSolrClient) c;
            nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, httpClient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
            nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps)));
        }
        AddUpdateCommand c = new AddUpdateCommand(null);
        c.solrDoc = sdoc("id", id.incrementAndGet());
        if (nodes.size() > 0) {
            params = new ModifiableSolrParams();
            cmdDistrib.distribAdd(c, nodes, params);
        }
    }
    nodes.clear();
    for (SolrClient c : clients) {
        HttpSolrClient httpClient = (HttpSolrClient) c;
        nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, httpClient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
        nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps)));
    }
    final AtomicInteger commits = new AtomicInteger();
    for (JettySolrRunner jetty : jettys) {
        CoreContainer cores = jetty.getCoreContainer();
        try (SolrCore core = cores.getCore("collection1")) {
            core.getUpdateHandler().registerCommitCallback(new SolrEventListener() {

                @Override
                public void init(NamedList args) {
                }

                @Override
                public void postSoftCommit() {
                }

                @Override
                public void postCommit() {
                    commits.incrementAndGet();
                }

                @Override
                public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) {
                }
            });
        }
    }
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    assertEquals(getShardCount(), commits.get());
    for (SolrClient c : clients) {
        NamedList<Object> resp = c.request(new LukeRequest());
        assertEquals("SOLR-3428: We only did adds - there should be no deletes", ((NamedList<Object>) resp.get("index")).get("numDocs"), ((NamedList<Object>) resp.get("index")).get("maxDoc"));
    }
    testMaxRetries();
    testOneRetry();
    testRetryNodeAgainstBadAddress();
    testRetryNodeWontRetrySocketError();
    testDistribOpenSearcher();
}

Example 10 with Node

use of org.apache.solr.update.SolrCmdDistributor.Node in project lucene-solr by apache.

the class SolrCmdDistributorTest method testRetryNodeAgainstBadAddress.

private void testRetryNodeAgainstBadAddress() throws SolrServerException, IOException {
    // Test RetryNode
    SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
    final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
    long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults().getNumFound();
    ArrayList<Node> nodes = new ArrayList<>();
    ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, "[ff01::114]:33332" + context, ZkStateReader.CORE_NAME_PROP, "");
    RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {

        @Override
        public boolean checkRetry() {
            ZkNodeProps leaderProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
            this.nodeProps = new ZkCoreNodeProps(leaderProps);
            return true;
        }
    };
    nodes.add(retryNode);
    AddUpdateCommand cmd = new AddUpdateCommand(null);
    cmd.solrDoc = sdoc("id", id.incrementAndGet());
    ModifiableSolrParams params = new ModifiableSolrParams();
    cmdDistrib.distribAdd(cmd, nodes, params);
    CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
    params = new ModifiableSolrParams();
    params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
    cmdDistrib.distribCommit(ccmd, nodes, params);
    cmdDistrib.finish();
    long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults().getNumFound();
    // different OS's will throw different exceptions for the bad address above
    if (numFoundBefore != numFoundAfter) {
        assertEquals(0, cmdDistrib.getErrors().size());
        assertEquals(numFoundBefore + 1, numFoundAfter);
    } else {
        // we will get java.net.SocketException: Network is unreachable and not retry
        assertEquals(numFoundBefore, numFoundAfter);
        assertEquals(1, cmdDistrib.getErrors().size());
    }
}