Examples with DocCollection org.apache.solr.common.cloud.DocCollection used on opensource projects

Example 71 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class TestSystemCollAutoCreate method testAutoCreate.

public void testAutoCreate() throws Exception {
    TestBlobHandler.checkBlobPost(cloudJettys.get(0).jetty.getBaseUrl().toExternalForm(), cloudClient);
    DocCollection sysColl = cloudClient.getZkStateReader().getClusterState().getCollection(".system");
}

Example 72 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class TestSolrConfigHandlerCloud method getRandomServer.

public static String getRandomServer(CloudSolrClient cloudClient, String collName) {
    DocCollection coll = cloudClient.getZkStateReader().getClusterState().getCollection(collName);
    List<String> urls = new ArrayList<>();
    for (Slice slice : coll.getSlices()) {
        for (Replica replica : slice.getReplicas()) urls.add("" + replica.get(ZkStateReader.BASE_URL_PROP) + "/" + replica.get(ZkStateReader.CORE_NAME_PROP));
    }
    return urls.get(random().nextInt(urls.size()));
}

Example 73 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class ShardSplitTest method testSplitStaticIndexReplication.

/*
  Creates a collection with replicationFactor=1, splits a shard. Restarts the sub-shard leader node.
  Add a replica. Ensure count matches in leader and replica.
   */
public void testSplitStaticIndexReplication() throws Exception {
    waitForThingsToLevelOut(15);
    DocCollection defCol = cloudClient.getZkStateReader().getClusterState().getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
    Replica replica = defCol.getReplicas().get(0);
    String nodeName = replica.getNodeName();
    String collectionName = "testSplitStaticIndexReplication";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, "conf1", 1, 1);
    // some high number so we can create replicas without hindrance
    create.setMaxShardsPerNode(5);
    // we want to create the leader on a fixed node so that we know which one to restart later
    create.setCreateNodeSet(nodeName);
    create.process(cloudClient);
    try (CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress(), true, cloudClient.getLbClient().getHttpClient())) {
        client.setDefaultCollection(collectionName);
        StoppableIndexingThread thread = new StoppableIndexingThread(controlClient, client, "i1", true);
        try {
            thread.start();
            // give the indexer sometime to do its work
            Thread.sleep(1000);
            thread.safeStop();
            thread.join();
            client.commit();
            controlClient.commit();
            CollectionAdminRequest.SplitShard splitShard = CollectionAdminRequest.splitShard(collectionName);
            splitShard.setShardName(SHARD1);
            String asyncId = splitShard.processAsync(client);
            RequestStatusState state = CollectionAdminRequest.requestStatus(asyncId).waitFor(client, 120);
            if (state == RequestStatusState.COMPLETED) {
                waitForRecoveriesToFinish(collectionName, true);
                // let's wait to see parent shard become inactive
                CountDownLatch latch = new CountDownLatch(1);
                client.getZkStateReader().registerCollectionStateWatcher(collectionName, new CollectionStateWatcher() {

                    @Override
                    public boolean onStateChanged(Set<String> liveNodes, DocCollection collectionState) {
                        Slice parent = collectionState.getSlice(SHARD1);
                        Slice slice10 = collectionState.getSlice(SHARD1_0);
                        Slice slice11 = collectionState.getSlice(SHARD1_1);
                        if (slice10 != null && slice11 != null && parent.getState() == Slice.State.INACTIVE && slice10.getState() == Slice.State.ACTIVE && slice11.getState() == Slice.State.ACTIVE) {
                            latch.countDown();
                            // removes the watch
                            return true;
                        }
                        return false;
                    }
                });
                latch.await(1, TimeUnit.MINUTES);
                if (latch.getCount() != 0) {
                    // sanity check
                    fail("Sub-shards did not become active even after waiting for 1 minute");
                }
                int liveNodeCount = client.getZkStateReader().getClusterState().getLiveNodes().size();
                // restart the sub-shard leader node
                boolean restarted = false;
                for (JettySolrRunner jetty : jettys) {
                    int port = jetty.getBaseUrl().getPort();
                    if (replica.getStr(BASE_URL_PROP).contains(":" + port)) {
                        ChaosMonkey.kill(jetty);
                        ChaosMonkey.start(jetty);
                        restarted = true;
                        break;
                    }
                }
                if (!restarted) {
                    // sanity check
                    fail("We could not find a jetty to kill for replica: " + replica.getCoreUrl());
                }
                // add a new replica for the sub-shard
                CollectionAdminRequest.AddReplica addReplica = CollectionAdminRequest.addReplicaToShard(collectionName, SHARD1_0);
                // use control client because less chances of it being the node being restarted
                // this is to avoid flakiness of test because of NoHttpResponseExceptions
                String control_collection = client.getZkStateReader().getClusterState().getCollection("control_collection").getReplicas().get(0).getStr(BASE_URL_PROP);
                try (HttpSolrClient control = new HttpSolrClient.Builder(control_collection).withHttpClient(client.getLbClient().getHttpClient()).build()) {
                    state = addReplica.processAndWait(control, 30);
                }
                if (state == RequestStatusState.COMPLETED) {
                    CountDownLatch newReplicaLatch = new CountDownLatch(1);
                    client.getZkStateReader().registerCollectionStateWatcher(collectionName, new CollectionStateWatcher() {

                        @Override
                        public boolean onStateChanged(Set<String> liveNodes, DocCollection collectionState) {
                            if (liveNodes.size() != liveNodeCount) {
                                return false;
                            }
                            Slice slice = collectionState.getSlice(SHARD1_0);
                            if (slice.getReplicas().size() == 2) {
                                if (!slice.getReplicas().stream().anyMatch(r -> r.getState() == Replica.State.RECOVERING)) {
                                    // we see replicas and none of them are recovering
                                    newReplicaLatch.countDown();
                                    return true;
                                }
                            }
                            return false;
                        }
                    });
                    newReplicaLatch.await(30, TimeUnit.SECONDS);
                    // check consistency of sub-shard replica explicitly because checkShardConsistency methods doesn't
                    // handle new shards/replica so well.
                    ClusterState clusterState = client.getZkStateReader().getClusterState();
                    DocCollection collection = clusterState.getCollection(collectionName);
                    int numReplicasChecked = assertConsistentReplicas(collection.getSlice(SHARD1_0));
                    assertEquals("We should have checked consistency for exactly 2 replicas of shard1_0", 2, numReplicasChecked);
                } else {
                    fail("Adding a replica to sub-shard did not complete even after waiting for 30 seconds!. Saw state = " + state.getKey());
                }
            } else {
                fail("We expected shard split to succeed on a static index but it didn't. Found state = " + state.getKey());
            }
        } finally {
            thread.safeStop();
            thread.join();
        }
    }
}

Example 74 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class ShardSplitTest method testSplitAfterFailedSplit.

/**
   * Used to test that we can split a shard when a previous split event
   * left sub-shards in construction or recovery state.
   *
   * See SOLR-9439
   */
@Test
public void testSplitAfterFailedSplit() throws Exception {
    waitForThingsToLevelOut(15);
    // we definitely want split to fail
    TestInjection.splitFailureBeforeReplicaCreation = "true:100";
    try {
        try {
            CollectionAdminRequest.SplitShard splitShard = CollectionAdminRequest.splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
            splitShard.setShardName(SHARD1);
            splitShard.process(cloudClient);
            fail("Shard split was not supposed to succeed after failure injection!");
        } catch (Exception e) {
        // expected
        }
        // assert that sub-shards cores exist and sub-shard is in construction state
        ZkStateReader zkStateReader = cloudClient.getZkStateReader();
        zkStateReader.forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
        ClusterState state = zkStateReader.getClusterState();
        DocCollection collection = state.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
        Slice shard10 = collection.getSlice(SHARD1_0);
        assertEquals(Slice.State.CONSTRUCTION, shard10.getState());
        assertEquals(1, shard10.getReplicas().size());
        Slice shard11 = collection.getSlice(SHARD1_1);
        assertEquals(Slice.State.CONSTRUCTION, shard11.getState());
        assertEquals(1, shard11.getReplicas().size());
        // lets retry the split
        // let the split succeed
        TestInjection.reset();
        try {
            CollectionAdminRequest.SplitShard splitShard = CollectionAdminRequest.splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION);
            splitShard.setShardName(SHARD1);
            splitShard.process(cloudClient);
        // Yay!
        } catch (Exception e) {
            log.error("Shard split failed", e);
            fail("Shard split did not succeed after a previous failed split attempt left sub-shards in construction state");
        }
    } finally {
        TestInjection.reset();
    }
}

Example 75 with DocCollection

use of org.apache.solr.common.cloud.DocCollection in project lucene-solr by apache.

the class DistribDocExpirationUpdateProcessorTest method getIndexVersionOfAllReplicas.

/**
   * returns a map whose key is the coreNodeName and whose value is what the replication
   * handler returns for the indexversion
   */
private Map<String, Long> getIndexVersionOfAllReplicas() throws IOException, SolrServerException {
    Map<String, Long> results = new HashMap<String, Long>();
    DocCollection collectionState = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION);
    for (Replica replica : collectionState.getReplicas()) {
        String coreName = replica.getCoreName();
        try (HttpSolrClient client = getHttpSolrClient(replica.getCoreUrl())) {
            ModifiableSolrParams params = new ModifiableSolrParams();
            params.set("command", "indexversion");
            params.set("_trace", "getIndexVersion");
            params.set("qt", ReplicationHandler.PATH);
            QueryRequest req = new QueryRequest(params);
            NamedList<Object> res = client.request(req);
            assertNotNull("null response from server: " + coreName, res);
            Object version = res.get("indexversion");
            assertNotNull("null version from server: " + coreName, version);
            assertTrue("version isn't a long: " + coreName, version instanceof Long);
            results.put(coreName, (Long) version);
            long numDocs = client.query(params("q", "*:*", "distrib", "false", "rows", "0", "_trace", "counting_docs")).getResults().getNumFound();
            log.info("core=" + coreName + "; ver=" + version + "; numDocs=" + numDocs);
        }
    }
    return results;
}