Examples with ZkStateReader org.apache.solr.common.cloud.ZkStateReader used on opensource projects

Example 76 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method createJettys.

protected List<JettySolrRunner> createJettys(int numJettys) throws Exception {
    List<JettySolrRunner> jettys = new ArrayList<>();
    List<SolrClient> clients = new ArrayList<>();
    StringBuilder sb = new StringBuilder();
    if ("2".equals(getStateFormat())) {
        log.info("Creating " + DEFAULT_COLLECTION + " with stateFormat=2");
        SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT);
        Overseer.getStateUpdateQueue(zkClient).offer(Utils.toJSON(Utils.makeMap(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), "name", DEFAULT_COLLECTION, "numShards", String.valueOf(sliceCount), DocCollection.STATE_FORMAT, getStateFormat(), ZkStateReader.NRT_REPLICAS, useTlogReplicas() ? "0" : "1", ZkStateReader.TLOG_REPLICAS, useTlogReplicas() ? "1" : "0", ZkStateReader.PULL_REPLICAS, String.valueOf(getPullReplicaCount()))));
        zkClient.close();
    }
    int numPullReplicas = getPullReplicaCount() * sliceCount;
    for (int i = 1; i <= numJettys; i++) {
        if (sb.length() > 0)
            sb.append(',');
        int cnt = this.jettyIntCntr.incrementAndGet();
        File jettyDir = createTempDir("shard-" + i).toFile();
        jettyDir.mkdirs();
        setupJettySolrHome(jettyDir);
        JettySolrRunner j;
        if (numPullReplicas > 0) {
            numPullReplicas--;
            log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.PULL);
            j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty" + cnt) : null, null, "solrconfig.xml", null, Replica.Type.PULL);
        } else if (useTlogReplicas()) {
            log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.TLOG);
            j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty" + cnt) : null, null, "solrconfig.xml", null, Replica.Type.TLOG);
        } else {
            log.info("create jetty {} in directory {} of type {}", i, jettyDir, Replica.Type.NRT);
            j = createJetty(jettyDir, useJettyDataDir ? getDataDir(testDir + "/jetty" + cnt) : null, null, "solrconfig.xml", null, null);
        }
        jettys.add(j);
        SolrClient client = createNewSolrClient(j.getLocalPort());
        clients.add(client);
    }
    this.jettys.addAll(jettys);
    this.clients.addAll(clients);
    int numReplicas = getTotalReplicas(DEFAULT_COLLECTION);
    int expectedNumReplicas = numJettys;
    // now wait until we see that the number of shards in the cluster state
    // matches what we expect
    int retries = 0;
    while (numReplicas != expectedNumReplicas) {
        numReplicas = getTotalReplicas(DEFAULT_COLLECTION);
        if (numReplicas == expectedNumReplicas)
            break;
        if (retries++ == 60) {
            printLayoutOnTearDown = true;
            fail("Number of replicas in the state does not match what we set:" + numReplicas + " vs " + expectedNumReplicas);
        }
        Thread.sleep(500);
    }
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    // make sure we have a leader for each shard
    for (int i = 1; i <= sliceCount; i++) {
        zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + i, 10000);
    }
    if (numReplicas > 0) {
        updateMappingsFromZk(this.jettys, this.clients);
    }
    // build the shard string
    for (int i = 1; i <= numJettys / 2; i++) {
        JettySolrRunner j = this.jettys.get(i);
        JettySolrRunner j2 = this.jettys.get(i + (numJettys / 2 - 1));
        if (sb.length() > 0)
            sb.append(',');
        sb.append(buildUrl(j.getLocalPort()));
        sb.append("|").append(buildUrl(j2.getLocalPort()));
    }
    shards = sb.toString();
    return jettys;
}

Example 77 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method checkShardConsistency.

/**
   * Returns a non-null string if replicas within the same shard do not have a
   * consistent number of documents.
   * If expectFailure==false, the exact differences found will be logged since
   * this would be an unexpected failure.
   * verbose causes extra debugging into to be displayed, even if everything is
   * consistent.
   */
protected String checkShardConsistency(String shard, boolean expectFailure, boolean verbose) throws Exception {
    List<CloudJettyRunner> solrJetties = shardToJetty.get(shard);
    if (solrJetties == null) {
        throw new RuntimeException("shard not found:" + shard + " keys:" + shardToJetty.keySet());
    }
    long num = -1;
    long lastNum = -1;
    String failMessage = null;
    if (verbose)
        System.err.println("check const of " + shard);
    int cnt = 0;
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    assertEquals("The client count does not match up with the shard count for slice:" + shard, zkStateReader.getClusterState().getSlice(DEFAULT_COLLECTION, shard).getReplicasMap().size(), solrJetties.size());
    CloudJettyRunner lastJetty = null;
    for (CloudJettyRunner cjetty : solrJetties) {
        ZkNodeProps props = cjetty.info;
        if (verbose)
            System.err.println("client" + cnt++);
        if (verbose)
            System.err.println("PROPS:" + props);
        try {
            // "tests" is just a tag that won't do anything except be echoed in logs
            SolrParams query = params("q", "*:*", "rows", "0", "distrib", "false", "tests", "checkShardConsistency");
            num = cjetty.client.solrClient.query(query).getResults().getNumFound();
        } catch (SolrServerException e) {
            if (verbose)
                System.err.println("error contacting client: " + e.getMessage() + "\n");
            continue;
        } catch (SolrException e) {
            if (verbose)
                System.err.println("error contacting client: " + e.getMessage() + "\n");
            continue;
        }
        boolean live = false;
        String nodeName = props.getStr(ZkStateReader.NODE_NAME_PROP);
        if (zkStateReader.getClusterState().liveNodesContain(nodeName)) {
            live = true;
        }
        if (verbose)
            System.err.println(" live:" + live);
        if (verbose)
            System.err.println(" num:" + num + "\n");
        boolean active = Replica.State.getState(props.getStr(ZkStateReader.STATE_PROP)) == Replica.State.ACTIVE;
        if (active && live) {
            if (lastNum > -1 && lastNum != num && failMessage == null) {
                failMessage = shard + " is not consistent.  Got " + lastNum + " from " + lastJetty.url + " (previous client)" + " and got " + num + " from " + cjetty.url;
                if (!expectFailure || verbose) {
                    System.err.println("######" + failMessage);
                    SolrQuery query = new SolrQuery("*:*");
                    query.set("distrib", false);
                    query.set("fl", "id,_version_");
                    query.set("rows", "100000");
                    query.set("sort", "id asc");
                    query.set("tests", "checkShardConsistency/showDiff");
                    SolrDocumentList lst1 = lastJetty.client.solrClient.query(query).getResults();
                    SolrDocumentList lst2 = cjetty.client.solrClient.query(query).getResults();
                    CloudInspectUtil.showDiff(lst1, lst2, lastJetty.url, cjetty.url);
                }
            }
            lastNum = num;
            lastJetty = cjetty;
        }
    }
    return failMessage;
}

Example 78 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method waitForRecoveriesToFinish.

protected void waitForRecoveriesToFinish(String collection, boolean verbose) throws Exception {
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    super.waitForRecoveriesToFinish(collection, zkStateReader, verbose);
}

Example 79 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class AbstractDistribZkTestBase method createServers.

@Override
protected void createServers(int numShards) throws Exception {
    // give everyone there own solrhome
    File controlHome = new File(new File(getSolrHome()).getParentFile(), "control" + homeCount.incrementAndGet());
    FileUtils.copyDirectory(new File(getSolrHome()), controlHome);
    setupJettySolrHome(controlHome);
    System.setProperty("collection", "control_collection");
    String numShardsS = System.getProperty(ZkStateReader.NUM_SHARDS_PROP);
    System.setProperty(ZkStateReader.NUM_SHARDS_PROP, "1");
    // let the shardId default to shard1
    controlJetty = createJetty(controlHome, null);
    System.clearProperty("collection");
    if (numShardsS != null) {
        System.setProperty(ZkStateReader.NUM_SHARDS_PROP, numShardsS);
    } else {
        System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
    }
    controlClient = createNewSolrClient(controlJetty.getLocalPort());
    StringBuilder sb = new StringBuilder();
    for (int i = 1; i <= numShards; i++) {
        if (sb.length() > 0)
            sb.append(',');
        // give everyone there own solrhome
        File jettyHome = new File(new File(getSolrHome()).getParentFile(), "jetty" + homeCount.incrementAndGet());
        setupJettySolrHome(jettyHome);
        JettySolrRunner j = createJetty(jettyHome, null, "shard" + (i + 2));
        jettys.add(j);
        clients.add(createNewSolrClient(j.getLocalPort()));
        sb.append(buildUrl(j.getLocalPort()));
    }
    shards = sb.toString();
    // now wait till we see the leader for each shard
    for (int i = 1; i <= numShards; i++) {
        ZkStateReader zkStateReader = jettys.get(0).getCoreContainer().getZkController().getZkStateReader();
        zkStateReader.getLeaderRetry("collection1", "shard" + (i + 2), 15000);
    }
}

Example 80 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method queryAndCompareReplicas.

/**
   * Executes a query against each live and active replica of the specified shard
   * and aserts that the results are identical.
   *
   * @see #queryAndCompare
   */
public QueryResponse queryAndCompareReplicas(SolrParams params, String shard) throws Exception {
    ArrayList<SolrClient> shardClients = new ArrayList<>(7);
    updateMappingsFromZk(jettys, clients);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    List<CloudJettyRunner> solrJetties = shardToJetty.get(shard);
    assertNotNull("no jetties found for shard: " + shard, solrJetties);
    for (CloudJettyRunner cjetty : solrJetties) {
        ZkNodeProps props = cjetty.info;
        String nodeName = props.getStr(ZkStateReader.NODE_NAME_PROP);
        boolean active = Replica.State.getState(props.getStr(ZkStateReader.STATE_PROP)) == Replica.State.ACTIVE;
        boolean live = zkStateReader.getClusterState().liveNodesContain(nodeName);
        if (active && live) {
            shardClients.add(cjetty.client.solrClient);
        }
    }
    return queryAndCompare(params, shardClients);
}