use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.
the class AbstractFullDistribZkTestBase method ensureAllReplicasAreActive.
protected List<Replica> ensureAllReplicasAreActive(String testCollectionName, String shardId, int shards, int rf, int maxWaitSecs) throws Exception {
final RTimer timer = new RTimer();
Map<String, Replica> notLeaders = new HashMap<>();
ZkStateReader zkr = cloudClient.getZkStateReader();
// force the state to be fresh
zkr.forceUpdateCollection(testCollectionName);
ClusterState cs = zkr.getClusterState();
Collection<Slice> slices = cs.getActiveSlices(testCollectionName);
assertTrue(slices.size() == shards);
boolean allReplicasUp = false;
long waitMs = 0L;
long maxWaitMs = maxWaitSecs * 1000L;
Replica leader = null;
while (waitMs < maxWaitMs && !allReplicasUp) {
cs = cloudClient.getZkStateReader().getClusterState();
assertNotNull(cs);
Slice shard = cs.getSlice(testCollectionName, shardId);
assertNotNull("No Slice for " + shardId, shard);
// assume true
allReplicasUp = true;
Collection<Replica> replicas = shard.getReplicas();
assertTrue("Did not find correct number of replicas. Expected:" + rf + " Found:" + replicas.size(), replicas.size() == rf);
leader = shard.getLeader();
assertNotNull(leader);
log.info("Found " + replicas.size() + " replicas and leader on " + leader.getNodeName() + " for " + shardId + " in " + testCollectionName);
// ensure all replicas are "active" and identify the non-leader replica
for (Replica replica : replicas) {
if (replica.getState() != Replica.State.ACTIVE) {
log.info("Replica {} is currently {}", replica.getName(), replica.getState());
allReplicasUp = false;
}
if (!leader.equals(replica))
notLeaders.put(replica.getName(), replica);
}
if (!allReplicasUp) {
try {
Thread.sleep(500L);
} catch (Exception ignoreMe) {
}
waitMs += 500L;
}
}
if (!allReplicasUp)
fail("Didn't see all replicas for shard " + shardId + " in " + testCollectionName + " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo());
if (notLeaders.isEmpty())
fail("Didn't isolate any replicas that are not the leader! ClusterState: " + printClusterStateInfo());
log.info("Took {} ms to see all replicas become active.", timer.getTime());
List<Replica> replicas = new ArrayList<>();
replicas.addAll(notLeaders.values());
return replicas;
}
use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.
the class AbstractFullDistribZkTestBase method checkCollectionExpectations.
private String checkCollectionExpectations(String collectionName, List<Integer> numShardsNumReplicaList, List<String> nodesAllowedToRunShards) {
ClusterState clusterState = getCommonCloudSolrClient().getZkStateReader().getClusterState();
int expectedSlices = numShardsNumReplicaList.get(0);
// The Math.min thing is here, because we expect replication-factor to be reduced to if there are not enough live nodes to spread all shards of a collection over different nodes
int expectedShardsPerSlice = numShardsNumReplicaList.get(1);
int expectedTotalShards = expectedSlices * expectedShardsPerSlice;
// .getCollectionStates();
if (clusterState.hasCollection(collectionName)) {
Map<String, Slice> slices = clusterState.getCollection(collectionName).getSlicesMap();
// did we find expectedSlices slices/shards?
if (slices.size() != expectedSlices) {
return "Found new collection " + collectionName + ", but mismatch on number of slices. Expected: " + expectedSlices + ", actual: " + slices.size();
}
int totalShards = 0;
for (String sliceName : slices.keySet()) {
for (Replica replica : slices.get(sliceName).getReplicas()) {
if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
return "Shard " + replica.getName() + " created on node " + replica.getNodeName() + " not allowed to run shards for the created collection " + collectionName;
}
}
totalShards += slices.get(sliceName).getReplicas().size();
}
if (totalShards != expectedTotalShards) {
return "Found new collection " + collectionName + " with correct number of slices, but mismatch on number of shards. Expected: " + expectedTotalShards + ", actual: " + totalShards;
}
return null;
} else {
return "Could not find new collection " + collectionName;
}
}
use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.
the class AbstractFullDistribZkTestBase method assertDocCounts.
protected void assertDocCounts(boolean verbose) throws Exception {
// and node/client to shard?
if (verbose)
System.err.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
long controlCount = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound();
// do some really inefficient mapping...
Map<String, Slice> slices = null;
ClusterState clusterState;
try (ZkStateReader zk = new ZkStateReader(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT)) {
zk.createClusterStateWatchersAndUpdate();
clusterState = zk.getClusterState();
slices = clusterState.getSlicesMap(DEFAULT_COLLECTION);
}
if (slices == null) {
throw new RuntimeException("Could not find collection " + DEFAULT_COLLECTION + " in " + clusterState.getCollectionsMap().keySet());
}
for (CloudJettyRunner cjetty : cloudJettys) {
CloudSolrServerClient client = cjetty.client;
for (Map.Entry<String, Slice> slice : slices.entrySet()) {
Map<String, Replica> theShards = slice.getValue().getReplicasMap();
for (Map.Entry<String, Replica> shard : theShards.entrySet()) {
String shardName = new URI(((HttpSolrClient) client.solrClient).getBaseURL()).getPort() + "_solr_";
if (verbose && shard.getKey().endsWith(shardName)) {
System.err.println("shard:" + slice.getKey());
System.err.println(shard.getValue());
}
}
}
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
long count = 0;
final Replica.State currentState = Replica.State.getState(cjetty.info.getStr(ZkStateReader.STATE_PROP));
if (currentState == Replica.State.ACTIVE && zkStateReader.getClusterState().liveNodesContain(cjetty.info.getStr(ZkStateReader.NODE_NAME_PROP))) {
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
count = client.solrClient.query(query).getResults().getNumFound();
}
if (verbose)
System.err.println("client docs:" + count + "\n\n");
}
if (verbose)
System.err.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
SolrQuery query = new SolrQuery("*:*");
assertEquals("Doc Counts do not add up", controlCount, cloudClient.query(query).getResults().getNumFound());
}
use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.
the class AbstractFullDistribZkTestBase method waitForReplicationFromReplicas.
protected void waitForReplicationFromReplicas(String collectionName, ZkStateReader zkStateReader, TimeOut timeout) throws KeeperException, InterruptedException, IOException {
zkStateReader.forceUpdateCollection(collectionName);
DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName);
Map<String, CoreContainer> containers = new HashMap<>();
for (JettySolrRunner runner : jettys) {
if (!runner.isRunning()) {
continue;
}
containers.put(runner.getNodeName(), runner.getCoreContainer());
}
for (Slice s : collection.getSlices()) {
Replica leader = s.getLeader();
long leaderIndexVersion = -1;
while (!timeout.hasTimedOut()) {
leaderIndexVersion = getIndexVersion(leader);
if (leaderIndexVersion >= 0) {
break;
}
Thread.sleep(1000);
}
if (timeout.hasTimedOut()) {
fail("Unable to get leader indexVersion");
}
for (Replica pullReplica : s.getReplicas(EnumSet.of(Replica.Type.PULL, Replica.Type.TLOG))) {
if (!zkStateReader.getClusterState().liveNodesContain(pullReplica.getNodeName())) {
continue;
}
while (true) {
long replicaIndexVersion = getIndexVersion(pullReplica);
if (leaderIndexVersion == replicaIndexVersion) {
log.debug("Leader replica's version ({}) in sync with replica({}): {} == {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion);
// Make sure the host is serving the correct version
try (SolrCore core = containers.get(pullReplica.getNodeName()).getCore(pullReplica.getCoreName())) {
RefCounted<SolrIndexSearcher> ref = core.getRegisteredSearcher();
try {
SolrIndexSearcher searcher = ref.get();
String servingVersion = searcher.getIndexReader().getIndexCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
if (Long.parseLong(servingVersion) == replicaIndexVersion) {
break;
} else {
log.debug("Replica {} has the correct version replicated, but the searcher is not ready yet. Replicated version: {}, Serving version: {}", pullReplica.getName(), replicaIndexVersion, servingVersion);
}
} finally {
if (ref != null)
ref.decref();
}
}
} else {
if (timeout.hasTimedOut()) {
logReplicaTypesReplicationInfo(collectionName, zkStateReader);
fail(String.format(Locale.ROOT, "Timed out waiting for replica %s (%d) to replicate from leader %s (%d)", pullReplica.getName(), replicaIndexVersion, leader.getName(), leaderIndexVersion));
}
if (leaderIndexVersion > replicaIndexVersion) {
log.debug("{} version is {} and leader's is {}, will wait for replication", pullReplica.getName(), replicaIndexVersion, leaderIndexVersion);
} else {
log.debug("Leader replica's version ({}) is lower than pull replica({}): {} < {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion);
}
}
Thread.sleep(1000);
}
}
}
}
use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.
the class AbstractFullDistribZkTestBase method updateMappingsFromZk.
protected void updateMappingsFromZk(List<JettySolrRunner> jettys, List<SolrClient> clients, boolean allowOverSharding) throws Exception {
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION);
cloudJettys.clear();
shardToJetty.clear();
ClusterState clusterState = zkStateReader.getClusterState();
DocCollection coll = clusterState.getCollection(DEFAULT_COLLECTION);
List<CloudSolrServerClient> theClients = new ArrayList<>();
for (SolrClient client : clients) {
// find info for this client in zk
nextClient: // we find out state by simply matching ports...
for (Slice slice : coll.getSlices()) {
for (Replica replica : slice.getReplicas()) {
int port = new URI(((HttpSolrClient) client).getBaseURL()).getPort();
if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
CloudSolrServerClient csc = new CloudSolrServerClient();
csc.solrClient = client;
csc.port = port;
csc.shardName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
csc.info = replica;
theClients.add(csc);
break nextClient;
}
}
}
}
for (JettySolrRunner jetty : jettys) {
int port = jetty.getLocalPort();
if (port == -1) {
throw new RuntimeException("Cannot find the port for jetty");
}
nextJetty: for (Slice slice : coll.getSlices()) {
Set<Entry<String, Replica>> entries = slice.getReplicasMap().entrySet();
for (Entry<String, Replica> entry : entries) {
Replica replica = entry.getValue();
if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
List<CloudJettyRunner> list = shardToJetty.get(slice.getName());
if (list == null) {
list = new ArrayList<>();
shardToJetty.put(slice.getName(), list);
}
boolean isLeader = slice.getLeader() == replica;
CloudJettyRunner cjr = new CloudJettyRunner();
cjr.jetty = jetty;
cjr.info = replica;
cjr.nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
cjr.coreNodeName = entry.getKey();
cjr.url = replica.getStr(ZkStateReader.BASE_URL_PROP) + "/" + replica.getStr(ZkStateReader.CORE_NAME_PROP);
cjr.client = findClientByPort(port, theClients);
list.add(cjr);
if (isLeader) {
shardToLeaderJetty.put(slice.getName(), cjr);
}
cloudJettys.add(cjr);
break nextJetty;
}
}
}
}
// running jetty though
for (Slice slice : coll.getSlices()) {
// check that things look right
List<CloudJettyRunner> jetties = shardToJetty.get(slice.getName());
if (!allowOverSharding) {
assertNotNull("Test setup problem: We found no jetties for shard: " + slice.getName() + " just:" + shardToJetty.keySet(), jetties);
assertEquals("slice:" + slice.getName(), slice.getReplicas().size(), jetties.size());
}
}
}
Aggregations