Examples with JettySolrRunner org.apache.solr.client.solrj.embedded.JettySolrRunner used on opensource projects

Example 21 with JettySolrRunner

use of org.apache.solr.client.solrj.embedded.JettySolrRunner in project lucene-solr by apache.

the class TestTlogReplica method testBasicLeaderElection.

public void testBasicLeaderElection() throws Exception {
    createAndWaitForCollection(1, 0, 2, 0);
    CloudSolrClient cloudClient = cluster.getSolrClient();
    new UpdateRequest().deleteByQuery("*:*").commit(cluster.getSolrClient(), collectionName);
    new UpdateRequest().add(sdoc("id", "1")).add(sdoc("id", "2")).process(cloudClient, collectionName);
    JettySolrRunner oldLeaderJetty = getSolrRunner(true).get(0);
    ChaosMonkey.kill(oldLeaderJetty);
    waitForState("Replica not removed", collectionName, activeReplicaCount(0, 1, 0));
    new UpdateRequest().add(sdoc("id", "3")).add(sdoc("id", "4")).process(cloudClient, collectionName);
    ChaosMonkey.start(oldLeaderJetty);
    waitForState("Replica not added", collectionName, activeReplicaCount(0, 2, 0));
    checkRTG(1, 4, cluster.getJettySolrRunners());
    new UpdateRequest().commit(cloudClient, collectionName);
    waitForNumDocsInAllActiveReplicas(4, 0);
}

Example 22 with JettySolrRunner

use of org.apache.solr.client.solrj.embedded.JettySolrRunner in project lucene-solr by apache.

the class TestTolerantUpdateProcessorCloud method createMiniSolrCloudCluster.

@BeforeClass
public static void createMiniSolrCloudCluster() throws Exception {
    final String configName = "solrCloudCollectionConfig";
    final File configDir = new File(TEST_HOME() + File.separator + "collection1" + File.separator + "conf");
    configureCluster(NUM_SERVERS).addConfig(configName, configDir.toPath()).configure();
    assertSpinLoopAllJettyAreRunning(cluster);
    CLOUD_CLIENT = cluster.getSolrClient();
    CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
    CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, NUM_SHARDS, REPLICATION_FACTOR).withProperty("config", "solrconfig-distrib-update-processor-chains.xml").withProperty("schema", // string id for doc routing prefix
    "schema15.xml").process(CLOUD_CLIENT);
    ZkStateReader zkStateReader = CLOUD_CLIENT.getZkStateReader();
    AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION_NAME, zkStateReader, true, true, 330);
    // really hackish way to get a URL for specific nodes based on shard/replica hosting
    // inspired by TestMiniSolrCloudCluster
    HashMap<String, String> urlMap = new HashMap<>();
    for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
        URL jettyURL = jetty.getBaseUrl();
        String nodeKey = jettyURL.getHost() + ":" + jettyURL.getPort() + jettyURL.getPath().replace("/", "_");
        urlMap.put(nodeKey, jettyURL.toString());
    }
    zkStateReader.updateClusterState();
    ClusterState clusterState = zkStateReader.getClusterState();
    for (Slice slice : clusterState.getSlices(COLLECTION_NAME)) {
        String shardName = slice.getName();
        Replica leader = slice.getLeader();
        assertNotNull("slice has null leader: " + slice.toString(), leader);
        assertNotNull("slice leader has null node name: " + slice.toString(), leader.getNodeName());
        String leaderUrl = urlMap.remove(leader.getNodeName());
        assertNotNull("could not find URL for " + shardName + " leader: " + leader.getNodeName(), leaderUrl);
        assertEquals("expected two total replicas for: " + slice.getName(), 2, slice.getReplicas().size());
        String passiveUrl = null;
        for (Replica replica : slice.getReplicas()) {
            if (!replica.equals(leader)) {
                passiveUrl = urlMap.remove(replica.getNodeName());
                assertNotNull("could not find URL for " + shardName + " replica: " + replica.getNodeName(), passiveUrl);
            }
        }
        assertNotNull("could not find URL for " + shardName + " replica", passiveUrl);
        if (shardName.equals("shard1")) {
            S_ONE_LEADER_CLIENT = getHttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/");
            S_ONE_NON_LEADER_CLIENT = getHttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/");
        } else if (shardName.equals("shard2")) {
            S_TWO_LEADER_CLIENT = getHttpSolrClient(leaderUrl + "/" + COLLECTION_NAME + "/");
            S_TWO_NON_LEADER_CLIENT = getHttpSolrClient(passiveUrl + "/" + COLLECTION_NAME + "/");
        } else {
            fail("unexpected shard: " + shardName);
        }
    }
    assertEquals("Should be exactly one server left (nost hosting either shard)", 1, urlMap.size());
    NO_COLLECTION_CLIENT = getHttpSolrClient(urlMap.values().iterator().next() + "/" + COLLECTION_NAME + "/");
    assertNotNull(S_ONE_LEADER_CLIENT);
    assertNotNull(S_TWO_LEADER_CLIENT);
    assertNotNull(S_ONE_NON_LEADER_CLIENT);
    assertNotNull(S_TWO_NON_LEADER_CLIENT);
    assertNotNull(NO_COLLECTION_CLIENT);
    // sanity check that our S_ONE_PRE & S_TWO_PRE really do map to shard1 & shard2 with default routing
    assertEquals(0, CLOUD_CLIENT.add(doc(f("id", S_ONE_PRE + random().nextInt()), f("expected_shard_s", "shard1"))).getStatus());
    assertEquals(0, CLOUD_CLIENT.add(doc(f("id", S_TWO_PRE + random().nextInt()), f("expected_shard_s", "shard2"))).getStatus());
    assertEquals(0, CLOUD_CLIENT.commit().getStatus());
    SolrDocumentList docs = CLOUD_CLIENT.query(params("q", "*:*", "fl", "id,expected_shard_s,[shard]")).getResults();
    assertEquals(2, docs.getNumFound());
    assertEquals(2, docs.size());
    for (SolrDocument doc : docs) {
        String expected = COLLECTION_NAME + "_" + doc.getFirstValue("expected_shard_s") + "_replica";
        String docShard = doc.getFirstValue("[shard]").toString();
        assertTrue("shard routing prefixes don't seem to be aligned anymore, " + "did someone change the default routing rules? " + "and/or the the default core name rules? " + "and/or the numShards used by this test? ... " + "couldn't find " + expected + " as substring of [shard] == '" + docShard + "' ... for docId == " + doc.getFirstValue("id"), docShard.contains(expected));
    }
}

Example 23 with JettySolrRunner

use of org.apache.solr.client.solrj.embedded.JettySolrRunner in project lucene-solr by apache.

the class TestRandomRequestDistribution method testQueryAgainstDownReplica.

/**
   * Asserts that requests against a collection are only served by a 'active' local replica
   */
private void testQueryAgainstDownReplica() throws Exception {
    log.info("Creating collection 'football' with 1 shard and 2 replicas");
    CollectionAdminRequest.createCollection("football", 1, 2).setCreateNodeSet(nodeNames.get(0) + ',' + nodeNames.get(1)).process(cloudClient);
    waitForRecoveriesToFinish("football", true);
    cloudClient.getZkStateReader().forceUpdateCollection("football");
    Replica leader = null;
    Replica notLeader = null;
    Collection<Replica> replicas = cloudClient.getZkStateReader().getClusterState().getSlice("football", "shard1").getReplicas();
    for (Replica replica : replicas) {
        if (replica.getStr(ZkStateReader.LEADER_PROP) != null) {
            leader = replica;
        } else {
            notLeader = replica;
        }
    }
    //Simulate a replica being in down state.
    ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), ZkStateReader.BASE_URL_PROP, notLeader.getStr(ZkStateReader.BASE_URL_PROP), ZkStateReader.NODE_NAME_PROP, notLeader.getStr(ZkStateReader.NODE_NAME_PROP), ZkStateReader.COLLECTION_PROP, "football", ZkStateReader.SHARD_ID_PROP, "shard1", ZkStateReader.CORE_NAME_PROP, notLeader.getStr(ZkStateReader.CORE_NAME_PROP), ZkStateReader.ROLES_PROP, "", ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
    log.info("Forcing {} to go into 'down' state", notLeader.getStr(ZkStateReader.CORE_NAME_PROP));
    DistributedQueue q = Overseer.getStateUpdateQueue(cloudClient.getZkStateReader().getZkClient());
    q.offer(Utils.toJSON(m));
    verifyReplicaStatus(cloudClient.getZkStateReader(), "football", "shard1", notLeader.getName(), Replica.State.DOWN);
    //Query against the node which hosts the down replica
    String baseUrl = notLeader.getStr(ZkStateReader.BASE_URL_PROP);
    if (!baseUrl.endsWith("/"))
        baseUrl += "/";
    String path = baseUrl + "football";
    log.info("Firing queries against path=" + path);
    try (HttpSolrClient client = getHttpSolrClient(path)) {
        client.setSoTimeout(5000);
        client.setConnectionTimeout(2000);
        SolrCore leaderCore = null;
        for (JettySolrRunner jetty : jettys) {
            CoreContainer container = jetty.getCoreContainer();
            for (SolrCore core : container.getCores()) {
                if (core.getName().equals(leader.getStr(ZkStateReader.CORE_NAME_PROP))) {
                    leaderCore = core;
                    break;
                }
            }
        }
        assertNotNull(leaderCore);
        SolrMetricManager leaderMetricManager = leaderCore.getCoreContainer().getMetricManager();
        String leaderRegistry = leaderCore.getCoreMetricManager().getRegistryName();
        Counter cnt = leaderMetricManager.counter(null, leaderRegistry, "requests", "QUERY.standard");
        // All queries should be served by the active replica
        // To make sure that's true we keep querying the down replica
        // If queries are getting processed by the down replica then the cluster state hasn't updated for that replica
        // locally
        // So we keep trying till it has updated and then verify if ALL queries go to the active replica
        long count = 0;
        while (true) {
            count++;
            client.query(new SolrQuery("*:*"));
            long c = cnt.getCount();
            if (c == 1) {
                // cluster state has got update locally
                break;
            } else {
                Thread.sleep(100);
            }
            if (count > 10000) {
                fail("After 10k queries we still see all requests being processed by the down replica");
            }
        }
        // Now we fire a few additional queries and make sure ALL of them
        // are served by the active replica
        int moreQueries = TestUtil.nextInt(random(), 4, 10);
        // Since 1 query has already hit the leader
        count = 1;
        for (int i = 0; i < moreQueries; i++) {
            client.query(new SolrQuery("*:*"));
            count++;
            long c = cnt.getCount();
            assertEquals("Query wasn't served by leader", count, c);
        }
    }
}

Example 24 with JettySolrRunner

use of org.apache.solr.client.solrj.embedded.JettySolrRunner in project lucene-solr by apache.

the class TestRandomRequestDistribution method testRequestTracking.

/**
   * Asserts that requests aren't always sent to the same poor node. See SOLR-7493
   */
private void testRequestTracking() throws Exception {
    CollectionAdminRequest.createCollection("a1x2", 1, 2).setCreateNodeSet(nodeNames.get(0) + ',' + nodeNames.get(1)).process(cloudClient);
    CollectionAdminRequest.createCollection("b1x1", 1, 1).setCreateNodeSet(nodeNames.get(2)).process(cloudClient);
    waitForRecoveriesToFinish("a1x2", true);
    waitForRecoveriesToFinish("b1x1", true);
    cloudClient.getZkStateReader().forceUpdateCollection("b1x1");
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    DocCollection b1x1 = clusterState.getCollection("b1x1");
    Collection<Replica> replicas = b1x1.getSlice("shard1").getReplicas();
    assertEquals(1, replicas.size());
    String baseUrl = replicas.iterator().next().getStr(ZkStateReader.BASE_URL_PROP);
    if (!baseUrl.endsWith("/"))
        baseUrl += "/";
    try (HttpSolrClient client = getHttpSolrClient(baseUrl + "a1x2")) {
        client.setSoTimeout(5000);
        client.setConnectionTimeout(2000);
        log.info("Making requests to " + baseUrl + "a1x2");
        for (int i = 0; i < 10; i++) {
            client.query(new SolrQuery("*:*"));
        }
    }
    Map<String, Integer> shardVsCount = new HashMap<>();
    for (JettySolrRunner runner : jettys) {
        CoreContainer container = runner.getCoreContainer();
        SolrMetricManager metricManager = container.getMetricManager();
        for (SolrCore core : container.getCores()) {
            String registry = core.getCoreMetricManager().getRegistryName();
            Counter cnt = metricManager.counter(null, registry, "requests", "QUERY.standard");
            SolrRequestHandler select = core.getRequestHandler("");
            //        long c = (long) select.getStatistics().get("requests");
            shardVsCount.put(core.getName(), (int) cnt.getCount());
        }
    }
    log.info("Shard count map = " + shardVsCount);
    for (Map.Entry<String, Integer> entry : shardVsCount.entrySet()) {
        assertTrue("Shard " + entry.getKey() + " received all 10 requests", entry.getValue() != 10);
    }
}

Example 25 with JettySolrRunner

use of org.apache.solr.client.solrj.embedded.JettySolrRunner in project lucene-solr by apache.

the class TestTlogReplica method checkRTG.

private void checkRTG(int from, int to, List<JettySolrRunner> solrRunners) throws Exception {
    for (JettySolrRunner solrRunner : solrRunners) {
        try (SolrClient client = solrRunner.newClient()) {
            for (int i = from; i <= to; i++) {
                SolrQuery query = new SolrQuery();
                query.set("distrib", false);
                query.setRequestHandler("/get");
                query.set("id", i);
                QueryResponse res = client.query(collectionName, query);
                assertNotNull("Can not find doc " + i + " in " + solrRunner.getBaseUrl(), res.getResponse().get("doc"));
            }
        }
    }
}