Examples with DocCollection org.apache.solr.common.cloud.DocCollection used on opensource projects

Example 91 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class DistributedUpdateProcessor method doDefensiveChecks.

private void doDefensiveChecks(DistribPhase phase) {
    boolean isReplayOrPeersync = (updateCommand.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
    if (isReplayOrPeersync)
        return;
    String from = req.getParams().get(DISTRIB_FROM);
    ClusterState clusterState = zkController.getClusterState();
    CloudDescriptor cloudDescriptor = req.getCore().getCoreDescriptor().getCloudDescriptor();
    DocCollection docCollection = clusterState.getCollection(collection);
    Slice mySlice = docCollection.getSlice(cloudDescriptor.getShardId());
    boolean localIsLeader = cloudDescriptor.isLeader();
    if (DistribPhase.FROMLEADER == phase && localIsLeader && from != null) {
        // from will be null on log replay
        String fromShard = req.getParams().get(DISTRIB_FROM_PARENT);
        if (fromShard != null) {
            if (mySlice.getState() == Slice.State.ACTIVE) {
                throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Request says it is coming from parent shard leader but we are in active state");
            }
            // shard splitting case -- check ranges to see if we are a sub-shard
            Slice fromSlice = docCollection.getSlice(fromShard);
            DocRouter.Range parentRange = fromSlice.getRange();
            if (parentRange == null)
                parentRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
            if (mySlice.getRange() != null && !mySlice.getRange().isSubsetOf(parentRange)) {
                throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Request says it is coming from parent shard leader but parent hash range is not superset of my range");
            }
        } else {
            // is it because of a routing rule?
            String fromCollection = req.getParams().get(DISTRIB_FROM_COLLECTION);
            if (fromCollection == null) {
                log.error("Request says it is coming from leader, but we are the leader: " + req.getParamString());
                SolrException solrExc = new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "Request says it is coming from leader, but we are the leader");
                solrExc.setMetadata("cause", "LeaderChanged");
                throw solrExc;
            }
        }
    }
    if ((isLeader && !localIsLeader) || (isSubShardLeader && !localIsLeader)) {
        log.error("ClusterState says we are the leader, but locally we don't think so");
        throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "ClusterState says we are the leader (" + zkController.getBaseUrl() + "/" + req.getCore().getName() + "), but locally we don't think so. Request came from " + from);
    }
}

Example 92 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class DistributedUpdateProcessor method setupRequest.

private List<Node> setupRequest(String id, SolrInputDocument doc, String route) {
    List<Node> nodes = null;
    // if we are in zk mode...
    if (zkEnabled) {
        assert TestInjection.injectUpdateRandomPause();
        if ((updateCommand.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0) {
            // we actually might be the leader, but we don't want leader-logic for these types of updates anyway.
            isLeader = false;
            forwardToLeader = false;
            return nodes;
        }
        ClusterState cstate = zkController.getClusterState();
        DocCollection coll = cstate.getCollection(collection);
        Slice slice = coll.getRouter().getTargetSlice(id, doc, route, req.getParams(), coll);
        if (slice == null) {
            // No slice found.  Most strict routers will have already thrown an exception, so a null return is
            // a signal to use the slice of this core.
            // TODO: what if this core is not in the targeted collection?
            String shardId = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
            slice = coll.getSlice(shardId);
            if (slice == null) {
                throw new SolrException(ErrorCode.BAD_REQUEST, "No shard " + shardId + " in " + coll);
            }
        }
        DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));
        if (DistribPhase.FROMLEADER == phase && !couldIbeSubShardLeader(coll)) {
            if (req.getCore().getCoreDescriptor().getCloudDescriptor().isLeader()) {
            // locally we think we are leader but the request says it came FROMLEADER
            // that could indicate a problem, let the full logic below figure it out
            } else {
                assert TestInjection.injectFailReplicaRequests();
                // we actually might be the leader, but we don't want leader-logic for these types of updates anyway.
                isLeader = false;
                forwardToLeader = false;
                return nodes;
            }
        }
        String shardId = slice.getName();
        try {
            // Not equivalent to getLeaderProps, which does retries to find a leader.
            // Replica leader = slice.getLeader();
            Replica leaderReplica = zkController.getZkStateReader().getLeaderRetry(collection, shardId);
            isLeader = leaderReplica.getName().equals(req.getCore().getCoreDescriptor().getCloudDescriptor().getCoreNodeName());
            if (!isLeader) {
                isSubShardLeader = amISubShardLeader(coll, slice, id, doc);
                if (isSubShardLeader) {
                    String myShardId = req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId();
                    slice = coll.getSlice(myShardId);
                    shardId = myShardId;
                    leaderReplica = zkController.getZkStateReader().getLeaderRetry(collection, myShardId);
                    List<ZkCoreNodeProps> myReplicas = zkController.getZkStateReader().getReplicaProps(collection, shardId, leaderReplica.getName(), null, Replica.State.DOWN);
                }
            }
            doDefensiveChecks(phase);
            // if request is coming from another collection then we want it to be sent to all replicas
            // even if its phase is FROMLEADER
            String fromCollection = updateCommand.getReq().getParams().get(DISTRIB_FROM_COLLECTION);
            if (DistribPhase.FROMLEADER == phase && !isSubShardLeader && fromCollection == null) {
                // we are coming from the leader, just go local - add no urls
                forwardToLeader = false;
            } else if (isLeader || isSubShardLeader) {
                // that means I want to forward onto my replicas...
                // so get the replicas...
                forwardToLeader = false;
                List<ZkCoreNodeProps> replicaProps = zkController.getZkStateReader().getReplicaProps(collection, shardId, leaderReplica.getName(), null, Replica.State.DOWN);
                if (replicaProps != null) {
                    if (nodes == null) {
                        nodes = new ArrayList<>(replicaProps.size());
                    }
                    // check for test param that lets us miss replicas
                    String[] skipList = req.getParams().getParams(TEST_DISTRIB_SKIP_SERVERS);
                    Set<String> skipListSet = null;
                    if (skipList != null) {
                        skipListSet = new HashSet<>(skipList.length);
                        skipListSet.addAll(Arrays.asList(skipList));
                        log.info("test.distrib.skip.servers was found and contains:" + skipListSet);
                    }
                    for (ZkCoreNodeProps props : replicaProps) {
                        if (skipList != null) {
                            boolean skip = skipListSet.contains(props.getCoreUrl());
                            log.info("check url:" + props.getCoreUrl() + " against:" + skipListSet + " result:" + skip);
                            if (!skip) {
                                nodes.add(new StdNode(props, collection, shardId));
                            }
                        } else {
                            nodes.add(new StdNode(props, collection, shardId));
                        }
                    }
                }
            } else {
                // I need to forward onto the leader...
                nodes = new ArrayList<>(1);
                nodes.add(new RetryNode(new ZkCoreNodeProps(leaderReplica), zkController.getZkStateReader(), collection, shardId));
                forwardToLeader = true;
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
        }
    }
    return nodes;
}

Example 93 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class ChaosMonkeyNothingIsSafeWithPullReplicasTest method test.

@Test
public void test() throws Exception {
    cloudClient.setSoTimeout(clientSoTimeout);
    DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION);
    assertEquals(this.sliceCount, docCollection.getSlices().size());
    Slice s = docCollection.getSlice("shard1");
    assertNotNull(s);
    assertEquals("Unexpected number of replicas. Collection: " + docCollection, numRealtimeOrTlogReplicas + numPullReplicas, s.getReplicas().size());
    assertEquals("Unexpected number of pull replicas. Collection: " + docCollection, numPullReplicas, s.getReplicas(EnumSet.of(Replica.Type.PULL)).size());
    assertEquals(useTlogReplicas() ? 0 : numRealtimeOrTlogReplicas, s.getReplicas(EnumSet.of(Replica.Type.NRT)).size());
    assertEquals(useTlogReplicas() ? numRealtimeOrTlogReplicas : 0, s.getReplicas(EnumSet.of(Replica.Type.TLOG)).size());
    boolean testSuccessful = false;
    try {
        handle.clear();
        handle.put("timestamp", SKIPVAL);
        ZkStateReader zkStateReader = cloudClient.getZkStateReader();
        // make sure we have leaders for each shard
        for (int j = 1; j < sliceCount; j++) {
            zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000);
        }
        // make sure we again have leaders for each shard
        waitForRecoveriesToFinish(false);
        // we cannot do delete by query
        // as it's not supported for recovery
        del("*:*");
        List<StoppableThread> threads = new ArrayList<>();
        List<StoppableIndexingThread> indexTreads = new ArrayList<>();
        int threadCount = TEST_NIGHTLY ? 3 : 1;
        int i = 0;
        for (i = 0; i < threadCount; i++) {
            StoppableIndexingThread indexThread = new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true);
            threads.add(indexThread);
            indexTreads.add(indexThread);
            indexThread.start();
        }
        threadCount = 1;
        i = 0;
        for (i = 0; i < threadCount; i++) {
            StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient);
            threads.add(searchThread);
            searchThread.start();
        }
        if (usually()) {
            StoppableCommitThread commitThread = new StoppableCommitThread(cloudClient, 1000, false);
            threads.add(commitThread);
            commitThread.start();
        }
        // TODO: we only do this sometimes so that we can sometimes compare against control,
        // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer
        boolean runFullThrottle = random().nextBoolean();
        if (runFullThrottle) {
            FullThrottleStoppableIndexingThread ftIndexThread = new FullThrottleStoppableIndexingThread(controlClient, cloudClient, clients, "ft1", true, this.clientSoTimeout);
            threads.add(ftIndexThread);
            ftIndexThread.start();
        }
        chaosMonkey.startTheMonkey(true, 10000);
        try {
            long runLength;
            if (RUN_LENGTH != -1) {
                runLength = RUN_LENGTH;
            } else {
                int[] runTimes;
                if (TEST_NIGHTLY) {
                    runTimes = new int[] { 5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000 };
                } else {
                    runTimes = new int[] { 5000, 7000, 15000 };
                }
                runLength = runTimes[random().nextInt(runTimes.length - 1)];
            }
            ChaosMonkey.wait(runLength, DEFAULT_COLLECTION, zkStateReader);
        } finally {
            chaosMonkey.stopTheMonkey();
        }
        // ideally this should go into chaosMonkey
        restartZk(1000 * (5 + random().nextInt(4)));
        for (StoppableThread indexThread : threads) {
            indexThread.safeStop();
        }
        // wait for stop...
        for (StoppableThread indexThread : threads) {
            indexThread.join();
        }
        // try and wait for any replications and what not to finish...
        ChaosMonkey.wait(2000, DEFAULT_COLLECTION, zkStateReader);
        // wait until there are no recoveries...
        //Math.round((runLength / 1000.0f / 3.0f)));
        waitForThingsToLevelOut(Integer.MAX_VALUE);
        // make sure we again have leaders for each shard
        for (int j = 1; j < sliceCount; j++) {
            zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000);
        }
        commit();
        // TODO: assert we didnt kill everyone
        zkStateReader.updateLiveNodes();
        assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0);
        // we expect full throttle fails, but cloud client should not easily fail
        for (StoppableThread indexThread : threads) {
            if (indexThread instanceof StoppableIndexingThread && !(indexThread instanceof FullThrottleStoppableIndexingThread)) {
                int failCount = ((StoppableIndexingThread) indexThread).getFailCount();
                assertFalse("There were too many update fails (" + failCount + " > " + FAIL_TOLERANCE + ") - we expect it can happen, but shouldn't easily", failCount > FAIL_TOLERANCE);
            }
        }
        waitForReplicationFromReplicas(DEFAULT_COLLECTION, zkStateReader, new TimeOut(30, TimeUnit.SECONDS));
        //      waitForAllWarmingSearchers();
        Set<String> addFails = getAddFails(indexTreads);
        Set<String> deleteFails = getDeleteFails(indexTreads);
        // full throttle thread can
        // have request fails
        checkShardConsistency(!runFullThrottle, true, addFails, deleteFails);
        long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
        // ensure we have added more than 0 docs
        long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound();
        assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0);
        if (VERBOSE)
            System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
        // sometimes we restart zookeeper as well
        if (random().nextBoolean()) {
            restartZk(1000 * (5 + random().nextInt(4)));
        }
        try (CloudSolrClient client = createCloudClient("collection1")) {
            // We don't really know how many live nodes we have at this point, so "maxShardsPerNode" needs to be > 1
            createCollection(null, "testcollection", 1, 1, 10, client, null, "conf1");
        }
        List<Integer> numShardsNumReplicas = new ArrayList<>(2);
        numShardsNumReplicas.add(1);
        numShardsNumReplicas.add(1 + getPullReplicaCount());
        checkForCollection("testcollection", numShardsNumReplicas, null);
        testSuccessful = true;
    } finally {
        if (!testSuccessful) {
            logReplicaTypesReplicationInfo(DEFAULT_COLLECTION, cloudClient.getZkStateReader());
            printLayout();
        }
    }
}

Example 94 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class ChaosMonkeySafeLeaderWithPullReplicasTest method test.

@Test
public void test() throws Exception {
    DocCollection docCollection = cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION);
    assertEquals(this.sliceCount, docCollection.getSlices().size());
    Slice s = docCollection.getSlice("shard1");
    assertNotNull(s);
    assertEquals("Unexpected number of replicas. Collection: " + docCollection, numRealtimeOrTlogReplicas + numPullReplicas, s.getReplicas().size());
    assertEquals("Unexpected number of pull replicas. Collection: " + docCollection, numPullReplicas, s.getReplicas(EnumSet.of(Replica.Type.PULL)).size());
    assertEquals(useTlogReplicas() ? 0 : numRealtimeOrTlogReplicas, s.getReplicas(EnumSet.of(Replica.Type.NRT)).size());
    assertEquals(useTlogReplicas() ? numRealtimeOrTlogReplicas : 0, s.getReplicas(EnumSet.of(Replica.Type.TLOG)).size());
    handle.clear();
    handle.put("timestamp", SKIPVAL);
    // randomly turn on 1 seconds 'soft' commit
    randomlyEnableAutoSoftCommit();
    tryDelete();
    List<StoppableThread> threads = new ArrayList<>();
    int threadCount = 2;
    int batchSize = 1;
    if (random().nextBoolean()) {
        batchSize = random().nextInt(98) + 2;
    }
    boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true;
    int maxUpdates = -1;
    if (!pauseBetweenUpdates) {
        maxUpdates = 1000 + random().nextInt(1000);
    } else {
        maxUpdates = 15000;
    }
    for (int i = 0; i < threadCount; i++) {
        // random().nextInt(999) + 1
        StoppableIndexingThread indexThread = new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true, maxUpdates, batchSize, pauseBetweenUpdates);
        threads.add(indexThread);
        indexThread.start();
    }
    StoppableCommitThread commitThread = new StoppableCommitThread(cloudClient, 1000, false);
    threads.add(commitThread);
    commitThread.start();
    chaosMonkey.startTheMonkey(false, 500);
    try {
        long runLength;
        if (RUN_LENGTH != -1) {
            runLength = RUN_LENGTH;
        } else {
            int[] runTimes;
            if (TEST_NIGHTLY) {
                runTimes = new int[] { 5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000 };
            } else {
                runTimes = new int[] { 5000, 7000, 15000 };
            }
            runLength = runTimes[random().nextInt(runTimes.length - 1)];
        }
        ChaosMonkey.wait(runLength, DEFAULT_COLLECTION, cloudClient.getZkStateReader());
    } finally {
        chaosMonkey.stopTheMonkey();
    }
    for (StoppableThread thread : threads) {
        thread.safeStop();
    }
    // wait for stop...
    for (StoppableThread thread : threads) {
        thread.join();
    }
    for (StoppableThread thread : threads) {
        if (thread instanceof StoppableIndexingThread) {
            assertEquals(0, ((StoppableIndexingThread) thread).getFailCount());
        }
    }
    // try and wait for any replications and what not to finish...
    Thread.sleep(2000);
    waitForThingsToLevelOut(180000);
    // even if things were leveled out, a jetty may have just been stopped or something
    // we wait again and wait to level out again to make sure the system is not still in flux
    Thread.sleep(3000);
    waitForThingsToLevelOut(180000);
    log.info("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
    waitForReplicationFromReplicas(DEFAULT_COLLECTION, cloudClient.getZkStateReader(), new TimeOut(30, TimeUnit.SECONDS));
    //    waitForAllWarmingSearchers();
    checkShardConsistency(batchSize == 1, true);
    // sometimes we restart zookeeper as well
    if (random().nextBoolean()) {
        zkServer.shutdown();
        zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort());
        zkServer.run();
    }
    try (CloudSolrClient client = createCloudClient("collection1")) {
        createCollection(null, "testcollection", 1, 1, 100, client, null, "conf1");
    }
    List<Integer> numShardsNumReplicas = new ArrayList<>(2);
    numShardsNumReplicas.add(1);
    numShardsNumReplicas.add(1 + getPullReplicaCount());
    checkForCollection("testcollection", numShardsNumReplicas, null);
}

Example 95 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class ClusterStateMockUtil method buildClusterState.

/**
   * This method lets you construct a complex ClusterState object by using simple strings of letters.
   *
   * c = collection, s = slice, r = replica, \d = node number (r2 means the replica is on node 2),
   * state = [A,R,D,F], * = replica to replace, binds to the left.
   *
   * For example:
   * csrr2rD*sr2csr
   *
   * Creates:
   *
   * 'csrr2rD*'
   * A collection, a shard, a replica on node 1 (the default) that is active (the default), a replica on node 2, and a replica on node 1
   * that has a state of down and is the replica we will be looking to put somewhere else (the *).
   *
   * 'sr2'
   * Then, another shard that has a replica on node 2.
   *
   * 'csr'
   * Then, another collection that has a shard with a single active replica on node 1.
   *
   * Result:
   *        {
   *         "collection2":{
   *           "maxShardsPerNode":"1",
   *           "replicationFactor":"1",
   *           "shards":{"slice1":{
   *               "state":"active",
   *               "replicas":{"replica5":{
   *                   "state":"active",
   *                   "node_name":"baseUrl1_",
   *                   "base_url":"http://baseUrl1"}}}}},
   *         "collection1":{
   *           "maxShardsPerNode":"1",
   *           "replicationFactor":"1",
   *           "shards":{
   *             "slice1":{
   *               "state":"active",
   *               "replicas":{
   *                 "replica3 (bad)":{
   *                   "state":"down",
   *                   "node_name":"baseUrl1_",
   *                   "base_url":"http://baseUrl1"},
   *                 "replica2":{
   *                   "state":"active",
   *                   "node_name":"baseUrl2_",
   *                   "base_url":"http://baseUrl2"},
   *                 "replica1":{
   *                   "state":"active",
   *                   "node_name":"baseUrl1_",
   *                   "base_url":"http://baseUrl1"}}},
   *             "slice2":{
   *               "state":"active",
   *               "replicas":{"replica4":{
   *                   "state":"active",
   *                   "node_name":"baseUrl2_",
   *                   "base_url":"http://baseUrl2"}}}}}}
   *
   */
@SuppressWarnings("resource")
protected static ClusterStateMockUtil.Result buildClusterState(List<Result> results, String clusterDescription, int replicationFactor, int maxShardsPerNode, String... liveNodes) {
    ClusterStateMockUtil.Result result = new ClusterStateMockUtil.Result();
    Map<String, Slice> slices = null;
    Map<String, Replica> replicas = null;
    Map<String, Object> collectionProps = new HashMap<>();
    collectionProps.put(ZkStateReader.MAX_SHARDS_PER_NODE, Integer.toString(maxShardsPerNode));
    collectionProps.put(ZkStateReader.REPLICATION_FACTOR, Integer.toString(replicationFactor));
    Map<String, DocCollection> collectionStates = new HashMap<>();
    DocCollection docCollection = null;
    Slice slice = null;
    int replicaCount = 1;
    Matcher m = BLUEPRINT.matcher(clusterDescription);
    while (m.find()) {
        Replica replica;
        switch(m.group(1)) {
            case "c":
                slices = new HashMap<>();
                docCollection = new DocCollection("collection" + (collectionStates.size() + 1), slices, collectionProps, null);
                collectionStates.put(docCollection.getName(), docCollection);
                break;
            case "s":
                replicas = new HashMap<>();
                slice = new Slice("slice" + (slices.size() + 1), replicas, null);
                slices.put(slice.getName(), slice);
                break;
            case "r":
                Map<String, Object> replicaPropMap = new HashMap<>();
                String node;
                node = m.group(2);
                if (node == null || node.trim().length() == 0) {
                    node = "1";
                }
                Replica.State state = Replica.State.ACTIVE;
                String stateCode = m.group(3);
                if (stateCode != null) {
                    switch(stateCode.charAt(0)) {
                        case 'S':
                            state = Replica.State.ACTIVE;
                            break;
                        case 'R':
                            state = Replica.State.RECOVERING;
                            break;
                        case 'D':
                            state = Replica.State.DOWN;
                            break;
                        case 'F':
                            state = Replica.State.RECOVERY_FAILED;
                            break;
                        default:
                            throw new IllegalArgumentException("Unexpected state for replica: " + stateCode);
                    }
                }
                String nodeName = "baseUrl" + node + "_";
                String replicaName = "replica" + replicaCount++;
                if ("*".equals(m.group(4))) {
                    replicaName += " (bad)";
                }
                replicaPropMap.put(ZkStateReader.NODE_NAME_PROP, nodeName);
                replicaPropMap.put(ZkStateReader.BASE_URL_PROP, "http://baseUrl" + node);
                replicaPropMap.put(ZkStateReader.STATE_PROP, state.toString());
                replica = new Replica(replicaName, replicaPropMap);
                if ("*".equals(m.group(4))) {
                    result.badReplica = new OverseerAutoReplicaFailoverThread.DownReplica();
                    result.badReplica.replica = replica;
                    result.badReplica.slice = slice;
                    result.badReplica.collection = docCollection;
                }
                replicas.put(replica.getName(), replica);
                break;
            default:
                break;
        }
    }
    ClusterState clusterState = new ClusterState(1, new HashSet<>(Arrays.asList(liveNodes)), collectionStates);
    MockZkStateReader reader = new MockZkStateReader(clusterState, collectionStates.keySet());
    String json;
    try {
        json = new String(Utils.toJSON(clusterState), "UTF-8");
    } catch (UnsupportedEncodingException e) {
        throw new RuntimeException("Unexpected");
    }
    System.err.println(json);
    // todo remove the limitation of always having a bad replica
    assert result.badReplica != null : "Is there no bad replica?";
    assert result.badReplica.slice != null : "Is there no bad replica?";
    result.reader = reader;
    if (results != null) {
        results.add(result);
    }
    return result;
}