Examples with ClusterState org.apache.solr.common.cloud.ClusterState used on opensource projects

Example 46 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class TestTolerantUpdateProcessorCloud method createMiniSolrCloudCluster.

@BeforeClass
public static void createMiniSolrCloudCluster() throws Exception {
    final String configName = "solrCloudCollectionConfig";
    final File configDir = new File(TEST_HOME() + File.separator + "collection1" + File.separator + "conf");
    configureCluster(NUM_SERVERS).addConfig(configName, configDir.toPath()).configure();
    assertSpinLoopAllJettyAreRunning(cluster);
    CLOUD_CLIENT = cluster.getSolrClient();
    CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
    CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, NUM_SHARDS, REPLICATION_FACTOR).withProperty("config", "solrconfig-distrib-update-processor-chains.xml").withProperty("schema", // string id for doc routing prefix
    "schema15.xml").process(CLOUD_CLIENT);
    ZkStateReader zkStateReader = CLOUD_CLIENT.getZkStateReader();
    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION_NAME, zkStateReader, true, true, 330);
    // really hackish way to get a URL for specific nodes based on shard/replica hosting
    // inspired by TestMiniSolrCloudCluster
    HashMap<String, String> urlMap = new HashMap<>();
    for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
        URL jettyURL = jetty.getBaseUrl();
        String nodeKey = jettyURL.getHost() + ":" + jettyURL.getPort() + jettyURL.getPath().replace("/", "_");
        urlMap.put(nodeKey, jettyURL.toString());
    }
    zkStateReader.updateClusterState();
    ClusterState clusterState = zkStateReader.getClusterState();
    for (Slice slice : clusterState.getSlices(COLLECTION_NAME)) {
        String shardName = slice.getName();
        Replica leader = slice.getLeader();
        assertNotNull("slice has null leader: " + slice.toString(), leader);
        assertNotNull("slice leader has null node name: " + slice.toString(), leader.getNodeName());
        String leaderUrl = urlMap.remove(leader.getNodeName());
        assertNotNull("could not find URL for " + shardName + " leader: " + leader.getNodeName(), leaderUrl);
        assertEquals("expected two total replicas for: " + slice.getName(), 2, slice.getReplicas().size());
        String passiveUrl = null;
        for (Replica replica : slice.getReplicas()) {
            if (!replica.equals(leader)) {
                passiveUrl = urlMap.remove(replica.getNodeName());
                assertNotNull("could not find URL for " + shardName + " replica: " + replica.getNodeName(), passiveUrl);
            }
        }
        assertNotNull("could not find URL for " + shardName + " replica", passiveUrl);
        if (shardName.equals("shard1")) {
            S_ONE_LEADER_CLIENT = getHttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/");
            S_ONE_NON_LEADER_CLIENT = getHttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/");
        } else if (shardName.equals("shard2")) {
            S_TWO_LEADER_CLIENT = getHttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/");
            S_TWO_NON_LEADER_CLIENT = getHttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/");
        } else {
            fail("unexpected shard: " + shardName);
        }
    }
    assertEquals("Should be exactly one server left (nost hosting either shard)", 1, urlMap.size());
    NO_COLLECTION_CLIENT = getHttpSolrClient(urlMap.values().iterator().next() + "/" + COLLECTION_NAME + "/");
    assertNotNull(S_ONE_LEADER_CLIENT);
    assertNotNull(S_TWO_LEADER_CLIENT);
    assertNotNull(S_ONE_NON_LEADER_CLIENT);
    assertNotNull(S_TWO_NON_LEADER_CLIENT);
    assertNotNull(NO_COLLECTION_CLIENT);
    // sanity check that our S_ONE_PRE & S_TWO_PRE really do map to shard1 & shard2 with default routing
    assertEquals(0, CLOUD_CLIENT.add(doc(f("id", S_ONE_PRE + random().nextInt()), f("expected_shard_s", "shard1"))).getStatus());
    assertEquals(0, CLOUD_CLIENT.add(doc(f("id", S_TWO_PRE + random().nextInt()), f("expected_shard_s", "shard2"))).getStatus());
    assertEquals(0, CLOUD_CLIENT.commit().getStatus());
    SolrDocumentList docs = CLOUD_CLIENT.query(params("q", "*:*", "fl", "id,expected_shard_s,[shard]")).getResults();
    assertEquals(2, docs.getNumFound());
    assertEquals(2, docs.size());
    for (SolrDocument doc : docs) {
        String expected = COLLECTION_NAME + "_" + doc.getFirstValue("expected_shard_s") + "_replica";
        String docShard = doc.getFirstValue("[shard]").toString();
        assertTrue("shard routing prefixes don't seem to be aligned anymore, " + "did someone change the default routing rules? " + "and/or the the default core name rules? " + "and/or the numShards used by this test? ... " + "couldn't find " + expected + " as substring of [shard] == '" + docShard + "' ... for docId == " + doc.getFirstValue("id"), docShard.contains(expected));
    }
}

Example 47 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class TestRandomRequestDistribution method testRequestTracking.

/**
   * Asserts that requests aren't always sent to the same poor node. See SOLR-7493
   */
private void testRequestTracking() throws Exception {
    CollectionAdminRequest.createCollection("a1x2", 1, 2).setCreateNodeSet(nodeNames.get(0) + ',' + nodeNames.get(1)).process(cloudClient);
    CollectionAdminRequest.createCollection("b1x1", 1, 1).setCreateNodeSet(nodeNames.get(2)).process(cloudClient);
    waitForRecoveriesToFinish("a1x2", true);
    waitForRecoveriesToFinish("b1x1", true);
    cloudClient.getZkStateReader().forceUpdateCollection("b1x1");
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    DocCollection b1x1 = clusterState.getCollection("b1x1");
    Collection<Replica> replicas = b1x1.getSlice("shard1").getReplicas();
    assertEquals(1, replicas.size());
    String baseUrl = replicas.iterator().next().getStr(ZkStateReader.BASE_URL_PROP);
    if (!baseUrl.endsWith("/"))
        baseUrl += "/";
    try (HttpSolrClient client = getHttpSolrClient(baseUrl + "a1x2")) {
        client.setSoTimeout(5000);
        client.setConnectionTimeout(2000);
        log.info("Making requests to " + baseUrl + "a1x2");
        for (int i = 0; i < 10; i++) {
            client.query(new SolrQuery("*:*"));
        }
    }
    Map<String, Integer> shardVsCount = new HashMap<>();
    for (JettySolrRunner runner : jettys) {
        CoreContainer container = runner.getCoreContainer();
        SolrMetricManager metricManager = container.getMetricManager();
        for (SolrCore core : container.getCores()) {
            String registry = core.getCoreMetricManager().getRegistryName();
            Counter cnt = metricManager.counter(null, registry, "requests", "QUERY.standard");
            SolrRequestHandler select = core.getRequestHandler("");
            //        long c = (long) select.getStatistics().get("requests");
            shardVsCount.put(core.getName(), (int) cnt.getCount());
        }
    }
    log.info("Shard count map = " + shardVsCount);
    for (Map.Entry<String, Integer> entry : shardVsCount.entrySet()) {
        assertTrue("Shard " + entry.getKey() + " received all 10 requests", entry.getValue() != 10);
    }
}

Example 48 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class TestStressInPlaceUpdates method getClientForLeader.

/**
   * Method gets the SolrClient for the leader replica. This is needed for a workaround for SOLR-8733.
   */
public SolrClient getClientForLeader() throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    cloudClient.getZkStateReader().forceUpdateCollection(DEFAULT_COLLECTION);
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    Replica leader = null;
    Slice shard1 = clusterState.getCollection(DEFAULT_COLLECTION).getSlice(SHARD1);
    leader = shard1.getLeader();
    for (int i = 0; i < clients.size(); i++) {
        String leaderBaseUrl = zkStateReader.getBaseUrlForNodeName(leader.getNodeName());
        if (((HttpSolrClient) clients.get(i)).getBaseURL().startsWith(leaderBaseUrl))
            return clients.get(i);
    }
    return null;
}

Example 49 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class TestCloudManagedSchemaConcurrent method verifyWaitForSchemaUpdateToPropagate.

private void verifyWaitForSchemaUpdateToPropagate() throws Exception {
    String testCollectionName = "collection1";
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    Replica shard1Leader = clusterState.getLeader(testCollectionName, "shard1");
    final String coreUrl = (new ZkCoreNodeProps(shard1Leader)).getCoreUrl();
    assertNotNull(coreUrl);
    RestTestHarness harness = new RestTestHarness(() -> coreUrl.endsWith("/") ? coreUrl.substring(0, coreUrl.length() - 1) : coreUrl);
    try {
        addFieldTypePut(harness, "fooInt", 15);
    } finally {
        harness.close();
    }
    // go into ZK to get the version of the managed schema after the update
    SolrZkClient zkClient = cloudClient.getZkStateReader().getZkClient();
    Stat stat = new Stat();
    String znodePath = "/configs/conf1/managed-schema";
    byte[] managedSchemaBytes = zkClient.getData(znodePath, null, stat, false);
    int schemaZkVersion = stat.getVersion();
    // now loop over all replicas and verify each has the same schema version
    Replica randomReplicaNotLeader = null;
    for (Slice slice : clusterState.getActiveSlices(testCollectionName)) {
        for (Replica replica : slice.getReplicas()) {
            validateZkVersion(replica, schemaZkVersion, 0, false);
            // save a random replica to test zk watcher behavior
            if (randomReplicaNotLeader == null && !replica.getName().equals(shard1Leader.getName()))
                randomReplicaNotLeader = replica;
        }
    }
    assertNotNull(randomReplicaNotLeader);
    // now update the data and then verify the znode watcher fires correctly
    // before an after a zk session expiration (see SOLR-6249)
    zkClient.setData(znodePath, managedSchemaBytes, schemaZkVersion, false);
    stat = new Stat();
    managedSchemaBytes = zkClient.getData(znodePath, null, stat, false);
    int updatedSchemaZkVersion = stat.getVersion();
    assertTrue(updatedSchemaZkVersion > schemaZkVersion);
    validateZkVersion(randomReplicaNotLeader, updatedSchemaZkVersion, 2, true);
    // ok - looks like the watcher fired correctly on the replica
    // now, expire that replica's zk session and then verify the watcher fires again (after reconnect)
    JettySolrRunner randomReplicaJetty = getJettyOnPort(getReplicaPort(randomReplicaNotLeader));
    assertNotNull(randomReplicaJetty);
    chaosMonkey.expireSession(randomReplicaJetty);
    // update the data again to cause watchers to fire
    zkClient.setData(znodePath, managedSchemaBytes, updatedSchemaZkVersion, false);
    stat = new Stat();
    managedSchemaBytes = zkClient.getData(znodePath, null, stat, false);
    updatedSchemaZkVersion = stat.getVersion();
    // give up to 10 secs for the replica to recover after zk session loss and see the update
    validateZkVersion(randomReplicaNotLeader, updatedSchemaZkVersion, 10, true);
}

Example 50 with ClusterState

use of org.apache.solr.common.cloud.ClusterState in project lucene-solr by apache.

the class TopicStream method constructStreams.

protected void constructStreams() throws IOException {
    try {
        ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
        Collection<Slice> slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);
        ModifiableSolrParams mParams = new ModifiableSolrParams(params);
        // We are the aggregator.
        mParams.set(DISTRIB, "false");
        String fl = mParams.get("fl");
        mParams.set(SORT, "_version_ asc");
        if (!fl.contains(VERSION_FIELD)) {
            fl += ",_version_";
        }
        mParams.set("fl", fl);
        Random random = new Random();
        ClusterState clusterState = zkStateReader.getClusterState();
        Set<String> liveNodes = clusterState.getLiveNodes();
        for (Slice slice : slices) {
            ModifiableSolrParams localParams = new ModifiableSolrParams(mParams);
            long checkpoint = checkpoints.get(slice.getName());
            Collection<Replica> replicas = slice.getReplicas();
            List<Replica> shuffler = new ArrayList<>();
            for (Replica replica : replicas) {
                if (replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName()))
                    shuffler.add(replica);
            }
            Replica rep = shuffler.get(random.nextInt(shuffler.size()));
            ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
            String url = zkProps.getCoreUrl();
            SolrStream solrStream = new SolrStream(url, localParams);
            solrStream.setSlice(slice.getName());
            solrStream.setCheckpoint(checkpoint);
            solrStream.setTrace(true);
            if (streamContext != null) {
                solrStream.setStreamContext(streamContext);
            }
            solrStreams.add(solrStream);
        }
    } catch (Exception e) {
        throw new IOException(e);
    }
}