Examples with ZkCoreNodeProps org.apache.solr.common.cloud.ZkCoreNodeProps used on opensource projects

Example 16 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class ShardSplitTest method checkDocCountsAndShardStates.

protected void checkDocCountsAndShardStates(int[] docCounts, int numReplicas) throws Exception {
    ClusterState clusterState = null;
    Slice slice1_0 = null, slice1_1 = null;
    int i = 0;
    for (i = 0; i < 10; i++) {
        ZkStateReader zkStateReader = cloudClient.getZkStateReader();
        clusterState = zkStateReader.getClusterState();
        slice1_0 = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, "shard1_0");
        slice1_1 = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, "shard1_1");
        if (slice1_0.getState() == Slice.State.ACTIVE && slice1_1.getState() == Slice.State.ACTIVE) {
            break;
        }
        Thread.sleep(500);
    }
    log.info("ShardSplitTest waited for {} ms for shard state to be set to active", i * 500);
    assertNotNull("Cluster state does not contain shard1_0", slice1_0);
    assertNotNull("Cluster state does not contain shard1_0", slice1_1);
    assertSame("shard1_0 is not active", Slice.State.ACTIVE, slice1_0.getState());
    assertSame("shard1_1 is not active", Slice.State.ACTIVE, slice1_1.getState());
    assertEquals("Wrong number of replicas created for shard1_0", numReplicas, slice1_0.getReplicas().size());
    assertEquals("Wrong number of replicas created for shard1_1", numReplicas, slice1_1.getReplicas().size());
    commit();
    // can't use checkShardConsistency because it insists on jettys and clients for each shard
    checkSubShardConsistency(SHARD1_0);
    checkSubShardConsistency(SHARD1_1);
    SolrQuery query = new SolrQuery("*:*").setRows(1000).setFields("id", "_version_");
    query.set("distrib", false);
    ZkCoreNodeProps shard1_0 = getLeaderUrlFromZk(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1_0);
    QueryResponse response;
    try (HttpSolrClient shard1_0Client = getHttpSolrClient(shard1_0.getCoreUrl())) {
        response = shard1_0Client.query(query);
    }
    long shard10Count = response.getResults().getNumFound();
    ZkCoreNodeProps shard1_1 = getLeaderUrlFromZk(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1_1);
    QueryResponse response2;
    try (HttpSolrClient shard1_1Client = getHttpSolrClient(shard1_1.getCoreUrl())) {
        response2 = shard1_1Client.query(query);
    }
    long shard11Count = response2.getResults().getNumFound();
    logDebugHelp(docCounts, response, shard10Count, response2, shard11Count);
    assertEquals("Wrong doc count on shard1_0. See SOLR-5309", docCounts[0], shard10Count);
    assertEquals("Wrong doc count on shard1_1. See SOLR-5309", docCounts[1], shard11Count);
}

Example 17 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class ShardSplitTest method checkSubShardConsistency.

protected void checkSubShardConsistency(String shard) throws SolrServerException, IOException {
    SolrQuery query = new SolrQuery("*:*").setRows(1000).setFields("id", "_version_");
    query.set("distrib", false);
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    Slice slice = clusterState.getSlice(AbstractDistribZkTestBase.DEFAULT_COLLECTION, shard);
    long[] numFound = new long[slice.getReplicasMap().size()];
    int c = 0;
    for (Replica replica : slice.getReplicas()) {
        String coreUrl = new ZkCoreNodeProps(replica).getCoreUrl();
        QueryResponse response;
        try (HttpSolrClient client = getHttpSolrClient(coreUrl)) {
            response = client.query(query);
        }
        numFound[c++] = response.getResults().getNumFound();
        log.info("Shard: " + shard + " Replica: {} has {} docs", coreUrl, String.valueOf(response.getResults().getNumFound()));
        assertTrue("Shard: " + shard + " Replica: " + coreUrl + " has 0 docs", response.getResults().getNumFound() > 0);
    }
    for (int i = 0; i < slice.getReplicasMap().size(); i++) {
        assertEquals(shard + " is not consistent", numFound[0], numFound[i]);
    }
}

Example 18 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class TestCloudManagedSchemaConcurrent method verifyWaitForSchemaUpdateToPropagate.

private void verifyWaitForSchemaUpdateToPropagate() throws Exception {
    String testCollectionName = "collection1";
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    Replica shard1Leader = clusterState.getLeader(testCollectionName, "shard1");
    final String coreUrl = (new ZkCoreNodeProps(shard1Leader)).getCoreUrl();
    assertNotNull(coreUrl);
    RestTestHarness harness = new RestTestHarness(() -> coreUrl.endsWith("/") ? coreUrl.substring(0, coreUrl.length() - 1) : coreUrl);
    try {
        addFieldTypePut(harness, "fooInt", 15);
    } finally {
        harness.close();
    }
    // go into ZK to get the version of the managed schema after the update
    SolrZkClient zkClient = cloudClient.getZkStateReader().getZkClient();
    Stat stat = new Stat();
    String znodePath = "/configs/conf1/managed-schema";
    byte[] managedSchemaBytes = zkClient.getData(znodePath, null, stat, false);
    int schemaZkVersion = stat.getVersion();
    // now loop over all replicas and verify each has the same schema version
    Replica randomReplicaNotLeader = null;
    for (Slice slice : clusterState.getActiveSlices(testCollectionName)) {
        for (Replica replica : slice.getReplicas()) {
            validateZkVersion(replica, schemaZkVersion, 0, false);
            // save a random replica to test zk watcher behavior
            if (randomReplicaNotLeader == null && !replica.getName().equals(shard1Leader.getName()))
                randomReplicaNotLeader = replica;
        }
    }
    assertNotNull(randomReplicaNotLeader);
    // now update the data and then verify the znode watcher fires correctly
    // before an after a zk session expiration (see SOLR-6249)
    zkClient.setData(znodePath, managedSchemaBytes, schemaZkVersion, false);
    stat = new Stat();
    managedSchemaBytes = zkClient.getData(znodePath, null, stat, false);
    int updatedSchemaZkVersion = stat.getVersion();
    assertTrue(updatedSchemaZkVersion > schemaZkVersion);
    validateZkVersion(randomReplicaNotLeader, updatedSchemaZkVersion, 2, true);
    // ok - looks like the watcher fired correctly on the replica
    // now, expire that replica's zk session and then verify the watcher fires again (after reconnect)
    JettySolrRunner randomReplicaJetty = getJettyOnPort(getReplicaPort(randomReplicaNotLeader));
    assertNotNull(randomReplicaJetty);
    chaosMonkey.expireSession(randomReplicaJetty);
    // update the data again to cause watchers to fire
    zkClient.setData(znodePath, managedSchemaBytes, updatedSchemaZkVersion, false);
    stat = new Stat();
    managedSchemaBytes = zkClient.getData(znodePath, null, stat, false);
    updatedSchemaZkVersion = stat.getVersion();
    // give up to 10 secs for the replica to recover after zk session loss and see the update
    validateZkVersion(randomReplicaNotLeader, updatedSchemaZkVersion, 10, true);
}

Example 19 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class TestCloudManagedSchemaConcurrent method validateZkVersion.

/**
   * Sends a GET request to get the zk schema version from a specific replica.
   */
protected void validateZkVersion(Replica replica, int schemaZkVersion, int waitSecs, boolean retry) throws Exception {
    final String replicaUrl = (new ZkCoreNodeProps(replica)).getCoreUrl();
    RestTestHarness testHarness = new RestTestHarness(() -> replicaUrl.endsWith("/") ? replicaUrl.substring(0, replicaUrl.length() - 1) : replicaUrl);
    try {
        long waitMs = waitSecs * 1000L;
        // wait a moment for the zk watcher to fire
        if (waitMs > 0)
            Thread.sleep(waitMs);
        try {
            testHarness.validateQuery("/schema/zkversion?wt=xml", "//zkversion=" + schemaZkVersion);
        } catch (Exception exc) {
            if (retry) {
                // brief wait before retrying
                Thread.sleep(waitMs > 0 ? waitMs : 2000L);
                testHarness.validateQuery("/schema/zkversion?wt=xml", "//zkversion=" + schemaZkVersion);
            } else {
                throw exc;
            }
        }
    } finally {
        testHarness.close();
    }
}

Example 20 with ZkCoreNodeProps

use of org.apache.solr.common.cloud.ZkCoreNodeProps in project lucene-solr by apache.

the class TopicStream method constructStreams.

protected void constructStreams() throws IOException {
    try {
        ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
        Collection<Slice> slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);
        ModifiableSolrParams mParams = new ModifiableSolrParams(params);
        // We are the aggregator.
        mParams.set(DISTRIB, "false");
        String fl = mParams.get("fl");
        mParams.set(SORT, "_version_ asc");
        if (!fl.contains(VERSION_FIELD)) {
            fl += ",_version_";
        }
        mParams.set("fl", fl);
        Random random = new Random();
        ClusterState clusterState = zkStateReader.getClusterState();
        Set<String> liveNodes = clusterState.getLiveNodes();
        for (Slice slice : slices) {
            ModifiableSolrParams localParams = new ModifiableSolrParams(mParams);
            long checkpoint = checkpoints.get(slice.getName());
            Collection<Replica> replicas = slice.getReplicas();
            List<Replica> shuffler = new ArrayList<>();
            for (Replica replica : replicas) {
                if (replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName()))
                    shuffler.add(replica);
            }
            Replica rep = shuffler.get(random.nextInt(shuffler.size()));
            ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
            String url = zkProps.getCoreUrl();
            SolrStream solrStream = new SolrStream(url, localParams);
            solrStream.setSlice(slice.getName());
            solrStream.setCheckpoint(checkpoint);
            solrStream.setTrace(true);
            if (streamContext != null) {
                solrStream.setStreamContext(streamContext);
            }
            solrStreams.add(solrStream);
        }
    } catch (Exception e) {
        throw new IOException(e);
    }
}