Examples with DocCollection org.apache.solr.common.cloud.DocCollection used on opensource projects

Example 56 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class TestConfigReload method checkConfReload.

private void checkConfReload(SolrZkClient client, String resPath, String name, String uri) throws Exception {
    Stat stat = new Stat();
    byte[] data = null;
    try {
        data = client.getData(resPath, null, stat, true);
    } catch (KeeperException.NoNodeException e) {
        data = "{}".getBytes(StandardCharsets.UTF_8);
        log.info("creating_node {}", resPath);
        client.create(resPath, data, CreateMode.PERSISTENT, true);
    }
    long startTime = System.nanoTime();
    Stat newStat = client.setData(resPath, data, true);
    client.setData("/configs/conf1", new byte[] { 1 }, true);
    assertTrue(newStat.getVersion() > stat.getVersion());
    log.info("new_version " + newStat.getVersion());
    Integer newVersion = newStat.getVersion();
    long maxTimeoutSeconds = 20;
    DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection("collection1");
    List<String> urls = new ArrayList<>();
    for (Slice slice : coll.getSlices()) {
        for (Replica replica : slice.getReplicas()) urls.add("" + replica.get(ZkStateReader.BASE_URL_PROP) + "/" + replica.get(ZkStateReader.CORE_NAME_PROP));
    }
    HashSet<String> succeeded = new HashSet<>();
    while (TimeUnit.SECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutSeconds) {
        Thread.sleep(50);
        for (String url : urls) {
            Map respMap = getAsMap(url + uri + "?wt=json");
            if (String.valueOf(newVersion).equals(String.valueOf(getObjectByPath(respMap, true, asList(name, "znodeVersion"))))) {
                succeeded.add(url);
            }
        }
        if (succeeded.size() == urls.size())
            break;
        succeeded.clear();
    }
    assertEquals(StrUtils.formatString("tried these servers {0} succeeded only in {1} ", urls, succeeded), urls.size(), succeeded.size());
}

Example 57 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method waitForReplicationFromReplicas.

protected void waitForReplicationFromReplicas(String collectionName, ZkStateReader zkStateReader, TimeOut timeout) throws KeeperException, InterruptedException, IOException {
    zkStateReader.forceUpdateCollection(collectionName);
    DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName);
    Map<String, CoreContainer> containers = new HashMap<>();
    for (JettySolrRunner runner : jettys) {
        if (!runner.isRunning()) {
            continue;
        }
        containers.put(runner.getNodeName(), runner.getCoreContainer());
    }
    for (Slice s : collection.getSlices()) {
        Replica leader = s.getLeader();
        long leaderIndexVersion = -1;
        while (!timeout.hasTimedOut()) {
            leaderIndexVersion = getIndexVersion(leader);
            if (leaderIndexVersion >= 0) {
                break;
            }
            Thread.sleep(1000);
        }
        if (timeout.hasTimedOut()) {
            fail("Unable to get leader indexVersion");
        }
        for (Replica pullReplica : s.getReplicas(EnumSet.of(Replica.Type.PULL, Replica.Type.TLOG))) {
            if (!zkStateReader.getClusterState().liveNodesContain(pullReplica.getNodeName())) {
                continue;
            }
            while (true) {
                long replicaIndexVersion = getIndexVersion(pullReplica);
                if (leaderIndexVersion == replicaIndexVersion) {
                    log.debug("Leader replica's version ({}) in sync with replica({}): {} == {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion);
                    // Make sure the host is serving the correct version
                    try (SolrCore core = containers.get(pullReplica.getNodeName()).getCore(pullReplica.getCoreName())) {
                        RefCounted<SolrIndexSearcher> ref = core.getRegisteredSearcher();
                        try {
                            SolrIndexSearcher searcher = ref.get();
                            String servingVersion = searcher.getIndexReader().getIndexCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
                            if (Long.parseLong(servingVersion) == replicaIndexVersion) {
                                break;
                            } else {
                                log.debug("Replica {} has the correct version replicated, but the searcher is not ready yet. Replicated version: {}, Serving version: {}", pullReplica.getName(), replicaIndexVersion, servingVersion);
                            }
                        } finally {
                            if (ref != null)
                                ref.decref();
                        }
                    }
                } else {
                    if (timeout.hasTimedOut()) {
                        logReplicaTypesReplicationInfo(collectionName, zkStateReader);
                        fail(String.format(Locale.ROOT, "Timed out waiting for replica %s (%d) to replicate from leader %s (%d)", pullReplica.getName(), replicaIndexVersion, leader.getName(), leaderIndexVersion));
                    }
                    if (leaderIndexVersion > replicaIndexVersion) {
                        log.debug("{} version is {} and leader's is {}, will wait for replication", pullReplica.getName(), replicaIndexVersion, leaderIndexVersion);
                    } else {
                        log.debug("Leader replica's version ({}) is lower than pull replica({}): {} < {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion);
                    }
                }
                Thread.sleep(1000);
            }
        }
    }
}

Example 58 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method updateMappingsFromZk.

protected void updateMappingsFromZk(List<JettySolrRunner> jettys, List<SolrClient> clients, boolean allowOverSharding) throws Exception {
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION);
    cloudJettys.clear();
    shardToJetty.clear();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection coll = clusterState.getCollection(DEFAULT_COLLECTION);
    List<CloudSolrServerClient> theClients = new ArrayList<>();
    for (SolrClient client : clients) {
        // find info for this client in zk 
        nextClient: // we find out state by simply matching ports...
        for (Slice slice : coll.getSlices()) {
            for (Replica replica : slice.getReplicas()) {
                int port = new URI(((HttpSolrClient) client).getBaseURL()).getPort();
                if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
                    CloudSolrServerClient csc = new CloudSolrServerClient();
                    csc.solrClient = client;
                    csc.port = port;
                    csc.shardName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
                    csc.info = replica;
                    theClients.add(csc);
                    break nextClient;
                }
            }
        }
    }
    for (JettySolrRunner jetty : jettys) {
        int port = jetty.getLocalPort();
        if (port == -1) {
            throw new RuntimeException("Cannot find the port for jetty");
        }
        nextJetty: for (Slice slice : coll.getSlices()) {
            Set<Entry<String, Replica>> entries = slice.getReplicasMap().entrySet();
            for (Entry<String, Replica> entry : entries) {
                Replica replica = entry.getValue();
                if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
                    List<CloudJettyRunner> list = shardToJetty.get(slice.getName());
                    if (list == null) {
                        list = new ArrayList<>();
                        shardToJetty.put(slice.getName(), list);
                    }
                    boolean isLeader = slice.getLeader() == replica;
                    CloudJettyRunner cjr = new CloudJettyRunner();
                    cjr.jetty = jetty;
                    cjr.info = replica;
                    cjr.nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
                    cjr.coreNodeName = entry.getKey();
                    cjr.url = replica.getStr(ZkStateReader.BASE_URL_PROP) + "/" + replica.getStr(ZkStateReader.CORE_NAME_PROP);
                    cjr.client = findClientByPort(port, theClients);
                    list.add(cjr);
                    if (isLeader) {
                        shardToLeaderJetty.put(slice.getName(), cjr);
                    }
                    cloudJettys.add(cjr);
                    break nextJetty;
                }
            }
        }
    }
    // running jetty though
    for (Slice slice : coll.getSlices()) {
        // check that things look right
        List<CloudJettyRunner> jetties = shardToJetty.get(slice.getName());
        if (!allowOverSharding) {
            assertNotNull("Test setup problem: We found no jetties for shard: " + slice.getName() + " just:" + shardToJetty.keySet(), jetties);
            assertEquals("slice:" + slice.getName(), slice.getReplicas().size(), jetties.size());
        }
    }
}

Example 59 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class AbstractDistribZkTestBase method waitForNewLeader.

static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader, TimeOut timeOut) throws Exception {
    log.info("Will wait for a node to become leader for {} secs", timeOut.timeLeft(SECONDS));
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION);
    for (; ; ) {
        ClusterState clusterState = zkStateReader.getClusterState();
        DocCollection coll = clusterState.getCollection("collection1");
        Slice slice = coll.getSlice(shardName);
        if (slice.getLeader() != null && !slice.getLeader().equals(oldLeader) && slice.getLeader().getState() == Replica.State.ACTIVE) {
            log.info("Old leader {}, new leader {}. New leader got elected in {} ms", oldLeader, slice.getLeader(), timeOut.timeElapsed(MILLISECONDS));
            break;
        }
        if (timeOut.hasTimedOut()) {
            Diagnostics.logThreadDumps("Could not find new leader in specified timeout");
            zkStateReader.getZkClient().printLayoutToStdOut();
            fail("Could not find new leader even after waiting for " + timeOut.timeElapsed(MILLISECONDS) + "ms");
        }
        Thread.sleep(100);
    }
}

Example 60 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method logReplicaTypesReplicationInfo.

protected void logReplicaTypesReplicationInfo(String collectionName, ZkStateReader zkStateReader) throws KeeperException, InterruptedException, IOException {
    log.info("## Collecting extra Replica.Type information of the cluster");
    zkStateReader.updateLiveNodes();
    StringBuilder builder = new StringBuilder();
    zkStateReader.forceUpdateCollection(collectionName);
    DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName);
    for (Slice s : collection.getSlices()) {
        Replica leader = s.getLeader();
        for (Replica r : s.getReplicas()) {
            if (!r.isActive(zkStateReader.getClusterState().getLiveNodes())) {
                builder.append(String.format(Locale.ROOT, "Replica %s not in liveNodes or is not active%s", r.getName(), System.lineSeparator()));
                continue;
            }
            if (r.equals(leader)) {
                builder.append(String.format(Locale.ROOT, "Replica %s is leader%s", r.getName(), System.lineSeparator()));
            }
            logReplicationDetails(r, builder);
        }
    }
    log.info("Summary of the cluster: " + builder.toString());
}