Examples with ClusterState org.apache.solr.common.cloud.ClusterState used on opensource projects

Example 26 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method checkCollectionExpectations.

private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
    ClusterState clusterState = getCommonCloudSolrClient().getZkStateReader().getClusterState();
    int expectedSlices = numShardsNumReplicaList.get(0);
    // The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
    int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
    int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
    //          .getCollectionStates();
    if (clusterState.hasCollection(collectionName)) {
        Map<String, Slice> slices = clusterState.getCollection(collectionName).getSlicesMap();
        // did we find expectedSlices slices/shards?
        if (slices.size() != expectedSlices) {
            return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
        }
        int totalShards = 0;
        for (String sliceName : slices.keySet()) {
            for (Replica replica : slices.get(sliceName).getReplicas()) {
                if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
                    return "Shard " + replica.getName() + " created on node " + replica.getNodeName() + " not allowed to run shards for the created collection " + collectionName;
                }
            }
            totalShards += slices.get(sliceName).getReplicas().size();
        }
        if (totalShards != expectedTotalShards) {
            return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
        }
        return null;
    } else {
        return "Could not find new collection " + collectionName;
    }
}

Example 27 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method assertDocCounts.

protected void assertDocCounts(boolean verbose) throws Exception {
    // and node/client to shard?
    if (verbose)
        System.err.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
    long controlCount = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
    // do some really inefficient mapping...
    Map<String, Slice> slices = null;
    ClusterState clusterState;
    try (ZkStateReader zk = new ZkStateReader(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT)) {
        zk.createClusterStateWatchersAndUpdate();
        clusterState = zk.getClusterState();
        slices = clusterState.getSlicesMap(DEFAULT_COLLECTION);
    }
    if (slices == null) {
        throw new RuntimeException("Could not find collection " + DEFAULT_COLLECTION + " in " + clusterState.getCollectionsMap().keySet());
    }
    for (CloudJettyRunner cjetty : cloudJettys) {
        CloudSolrServerClient client = cjetty.client;
        for (Map.Entry<String, Slice> slice : slices.entrySet()) {
            Map<String, Replica> theShards = slice.getValue().getReplicasMap();
            for (Map.Entry<String, Replica> shard : theShards.entrySet()) {
                String shardName = new URI(((HttpSolrClient) client.solrClient).getBaseURL()).getPort() + "_solr_";
                if (verbose && shard.getKey().endsWith(shardName)) {
                    System.err.println("shard:" + slice.getKey());
                    System.err.println(shard.getValue());
                }
            }
        }
        ZkStateReader zkStateReader = cloudClient.getZkStateReader();
        long count = 0;
        final Replica.State currentState = Replica.State.getState(cjetty.info.getStr(ZkStateReader.STATE_PROP));
        if (currentState == Replica.State.ACTIVE && zkStateReader.getClusterState().liveNodesContain(cjetty.info.getStr(ZkStateReader.NODE_NAME_PROP))) {
            SolrQuery query = new SolrQuery("*:*");
            query.set("distrib", false);
            count = client.solrClient.query(query).getResults().getNumFound();
        }
        if (verbose)
            System.err.println("client docs:" + count + "\n\n");
    }
    if (verbose)
        System.err.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
    SolrQuery query = new SolrQuery("*:*");
    assertEquals("Doc Counts do not add up", controlCount, cloudClient.query(query).getResults().getNumFound());
}

Example 28 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method updateMappingsFromZk.

protected void updateMappingsFromZk(List<JettySolrRunner> jettys, List<SolrClient> clients, boolean allowOverSharding) throws Exception {
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION);
    cloudJettys.clear();
    shardToJetty.clear();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection coll = clusterState.getCollection(DEFAULT_COLLECTION);
    List<CloudSolrServerClient> theClients = new ArrayList<>();
    for (SolrClient client : clients) {
        // find info for this client in zk 
        nextClient: // we find out state by simply matching ports...
        for (Slice slice : coll.getSlices()) {
            for (Replica replica : slice.getReplicas()) {
                int port = new URI(((HttpSolrClient) client).getBaseURL()).getPort();
                if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
                    CloudSolrServerClient csc = new CloudSolrServerClient();
                    csc.solrClient = client;
                    csc.port = port;
                    csc.shardName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
                    csc.info = replica;
                    theClients.add(csc);
                    break nextClient;
                }
            }
        }
    }
    for (JettySolrRunner jetty : jettys) {
        int port = jetty.getLocalPort();
        if (port == -1) {
            throw new RuntimeException("Cannot find the port for jetty");
        }
        nextJetty: for (Slice slice : coll.getSlices()) {
            Set<Entry<String, Replica>> entries = slice.getReplicasMap().entrySet();
            for (Entry<String, Replica> entry : entries) {
                Replica replica = entry.getValue();
                if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
                    List<CloudJettyRunner> list = shardToJetty.get(slice.getName());
                    if (list == null) {
                        list = new ArrayList<>();
                        shardToJetty.put(slice.getName(), list);
                    }
                    boolean isLeader = slice.getLeader() == replica;
                    CloudJettyRunner cjr = new CloudJettyRunner();
                    cjr.jetty = jetty;
                    cjr.info = replica;
                    cjr.nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
                    cjr.coreNodeName = entry.getKey();
                    cjr.url = replica.getStr(ZkStateReader.BASE_URL_PROP) + "/" + replica.getStr(ZkStateReader.CORE_NAME_PROP);
                    cjr.client = findClientByPort(port, theClients);
                    list.add(cjr);
                    if (isLeader) {
                        shardToLeaderJetty.put(slice.getName(), cjr);
                    }
                    cloudJettys.add(cjr);
                    break nextJetty;
                }
            }
        }
    }
    // running jetty though
    for (Slice slice : coll.getSlices()) {
        // check that things look right
        List<CloudJettyRunner> jetties = shardToJetty.get(slice.getName());
        if (!allowOverSharding) {
            assertNotNull("Test setup problem: We found no jetties for shard: " + slice.getName() + " just:" + shardToJetty.keySet(), jetties);
            assertEquals("slice:" + slice.getName(), slice.getReplicas().size(), jetties.size());
        }
    }
}

Example 29 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class AbstractDistribZkTestBase method waitForRecoveriesToFinish.

public static void waitForRecoveriesToFinish(String collection, ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout, int timeoutSeconds) throws Exception {
    log.info("Wait for recoveries to finish - collection: " + collection + " failOnTimeout:" + failOnTimeout + " timeout (sec):" + timeoutSeconds);
    boolean cont = true;
    int cnt = 0;
    while (cont) {
        if (verbose)
            System.out.println("-");
        boolean sawLiveRecovering = false;
        ClusterState clusterState = zkStateReader.getClusterState();
        Map<String, Slice> slices = clusterState.getSlicesMap(collection);
        assertNotNull("Could not find collection:" + collection, slices);
        for (Map.Entry<String, Slice> entry : slices.entrySet()) {
            Slice slice = entry.getValue();
            if (slice.getState() == Slice.State.CONSTRUCTION) {
                // similar to replica recovering; pretend its the same thing
                if (verbose)
                    System.out.println("Found a slice in construction state; will wait.");
                sawLiveRecovering = true;
            }
            Map<String, Replica> shards = slice.getReplicasMap();
            for (Map.Entry<String, Replica> shard : shards.entrySet()) {
                if (verbose)
                    System.out.println("replica:" + shard.getValue().getName() + " rstate:" + shard.getValue().getStr(ZkStateReader.STATE_PROP) + " live:" + clusterState.liveNodesContain(shard.getValue().getNodeName()));
                final Replica.State state = shard.getValue().getState();
                if ((state == Replica.State.RECOVERING || state == Replica.State.DOWN || state == Replica.State.RECOVERY_FAILED) && clusterState.liveNodesContain(shard.getValue().getStr(ZkStateReader.NODE_NAME_PROP))) {
                    sawLiveRecovering = true;
                }
            }
        }
        if (!sawLiveRecovering || cnt == timeoutSeconds) {
            if (!sawLiveRecovering) {
                if (verbose)
                    System.out.println("no one is recoverying");
            } else {
                if (verbose)
                    System.out.println("Gave up waiting for recovery to finish..");
                if (failOnTimeout) {
                    Diagnostics.logThreadDumps("Gave up waiting for recovery to finish.  THREAD DUMP:");
                    zkStateReader.getZkClient().printLayoutToStdOut();
                    fail("There are still nodes recoverying - waited for " + timeoutSeconds + " seconds");
                    // won't get here
                    return;
                }
            }
            cont = false;
        } else {
            Thread.sleep(1000);
        }
        cnt++;
    }
    log.info("Recoveries finished - collection: " + collection);
}

Example 30 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class AbstractDistribZkTestBase method waitForNewLeader.

static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader, TimeOut timeOut) throws Exception {
    log.info("Will wait for a node to become leader for {} secs", timeOut.timeLeft(SECONDS));
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION);
    for (; ; ) {
        ClusterState clusterState = zkStateReader.getClusterState();
        DocCollection coll = clusterState.getCollection("collection1");
        Slice slice = coll.getSlice(shardName);
        if (slice.getLeader() != null && !slice.getLeader().equals(oldLeader) && slice.getLeader().getState() == Replica.State.ACTIVE) {
            log.info("Old leader {}, new leader {}. New leader got elected in {} ms", oldLeader, slice.getLeader(), timeOut.timeElapsed(MILLISECONDS));
            break;
        }
        if (timeOut.hasTimedOut()) {
            Diagnostics.logThreadDumps("Could not find new leader in specified timeout");
            zkStateReader.getZkClient().printLayoutToStdOut();
            fail("Could not find new leader even after waiting for " + timeOut.timeElapsed(MILLISECONDS) + "ms");
        }
        Thread.sleep(100);
    }
}