Examples with Slice org.apache.solr.common.cloud.Slice used on opensource projects

Example 46 with Slice

use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.

the class ManagedIndexSchema method getActiveReplicaCoreUrls.

protected static List<String> getActiveReplicaCoreUrls(ZkController zkController, String collection, String localCoreNodeName) {
    List<String> activeReplicaCoreUrls = new ArrayList<>();
    ZkStateReader zkStateReader = zkController.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();
    Collection<Slice> activeSlices = clusterState.getActiveSlices(collection);
    if (activeSlices != null && activeSlices.size() > 0) {
        for (Slice next : activeSlices) {
            Map<String, Replica> replicasMap = next.getReplicasMap();
            if (replicasMap != null) {
                for (Map.Entry<String, Replica> entry : replicasMap.entrySet()) {
                    Replica replica = entry.getValue();
                    if (!localCoreNodeName.equals(replica.getName()) && replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
                        ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(replica);
                        activeReplicaCoreUrls.add(replicaCoreProps.getCoreUrl());
                    }
                }
            }
        }
    }
    return activeReplicaCoreUrls;
}

Example 47 with Slice

use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.

the class HttpSolrCall method getRemotCoreUrl.

private String getRemotCoreUrl(String collectionName, String origCorename) {
    ClusterState clusterState = cores.getZkController().getClusterState();
    Collection<Slice> slices = clusterState.getActiveSlices(collectionName);
    boolean byCoreName = false;
    if (slices == null) {
        slices = new ArrayList<>();
        // look by core name
        byCoreName = true;
        getSlicesForCollections(clusterState, slices, true);
        if (slices.isEmpty()) {
            getSlicesForCollections(clusterState, slices, false);
        }
    }
    if (slices.isEmpty()) {
        return null;
    }
    if (collectionsList == null)
        collectionsList = new ArrayList<>();
    collectionsList.add(collectionName);
    String coreUrl = getCoreUrl(collectionName, origCorename, clusterState, slices, byCoreName, true);
    if (coreUrl == null) {
        coreUrl = getCoreUrl(collectionName, origCorename, clusterState, slices, byCoreName, false);
    }
    return coreUrl;
}

Example 48 with Slice

use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.

the class CloudSolrClientTest method stateVersionParamTest.

@Test
public void stateVersionParamTest() throws Exception {
    DocCollection coll = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(COLLECTION);
    Replica r = coll.getSlices().iterator().next().getReplicas().iterator().next();
    SolrQuery q = new SolrQuery().setQuery("*:*");
    HttpSolrClient.RemoteSolrException sse = null;
    final String url = r.getStr(ZkStateReader.BASE_URL_PROP) + "/" + COLLECTION;
    try (HttpSolrClient solrClient = getHttpSolrClient(url)) {
        log.info("should work query, result {}", solrClient.query(q));
        //no problem
        q.setParam(CloudSolrClient.STATE_VERSION, COLLECTION + ":" + coll.getZNodeVersion());
        log.info("2nd query , result {}", solrClient.query(q));
        //no error yet good
        //an older version expect error
        q.setParam(CloudSolrClient.STATE_VERSION, COLLECTION + ":" + (coll.getZNodeVersion() - 1));
        QueryResponse rsp = solrClient.query(q);
        Map m = (Map) rsp.getResponse().get(CloudSolrClient.STATE_VERSION, rsp.getResponse().size() - 1);
        assertNotNull("Expected an extra information from server with the list of invalid collection states", m);
        assertNotNull(m.get(COLLECTION));
    }
    //now send the request to another node that does not serve the collection
    Set<String> allNodesOfColl = new HashSet<>();
    for (Slice slice : coll.getSlices()) {
        for (Replica replica : slice.getReplicas()) {
            allNodesOfColl.add(replica.getStr(ZkStateReader.BASE_URL_PROP));
        }
    }
    String theNode = null;
    Set<String> liveNodes = cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes();
    for (String s : liveNodes) {
        String n = cluster.getSolrClient().getZkStateReader().getBaseUrlForNodeName(s);
        if (!allNodesOfColl.contains(n)) {
            theNode = n;
            break;
        }
    }
    log.info("the node which does not serve this collection{} ", theNode);
    assertNotNull(theNode);
    final String solrClientUrl = theNode + "/" + COLLECTION;
    try (SolrClient solrClient = getHttpSolrClient(solrClientUrl)) {
        q.setParam(CloudSolrClient.STATE_VERSION, COLLECTION + ":" + (coll.getZNodeVersion() - 1));
        try {
            QueryResponse rsp = solrClient.query(q);
            log.info("error was expected");
        } catch (HttpSolrClient.RemoteSolrException e) {
            sse = e;
        }
        assertNotNull(sse);
        assertEquals(" Error code should be 510", SolrException.ErrorCode.INVALID_STATE.code, sse.code());
    }
}

Example 49 with Slice

use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.

the class ReplaceNodeTest method test.

@Test
public void test() throws Exception {
    cluster.waitForAllNodes(5000);
    String coll = "replacenodetest_coll";
    log.info("total_jettys: " + cluster.getJettySolrRunners().size());
    CloudSolrClient cloudClient = cluster.getSolrClient();
    Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
    ArrayList<String> l = new ArrayList<>(liveNodes);
    Collections.shuffle(l, random());
    String emptyNode = l.remove(0);
    String node2bdecommissioned = l.get(0);
    CollectionAdminRequest.Create create;
    // NOTE: always using the createCollection that takes in 'int' for all types of replicas, so we never
    // have to worry about null checking when comparing the Create command with the final Slices
    create = pickRandom(CollectionAdminRequest.createCollection(coll, "conf1", 5, 2, 0, 0), CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 1, 0), CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 1, 1), CollectionAdminRequest.createCollection(coll, "conf1", 5, 1, 0, 1), CollectionAdminRequest.createCollection(coll, "conf1", 5, 0, 2, 0));
    create.setCreateNodeSet(StrUtils.join(l, ',')).setMaxShardsPerNode(3);
    cloudClient.request(create);
    log.info("excluded_node : {}  ", emptyNode);
    new CollectionAdminRequest.ReplaceNode(node2bdecommissioned, emptyNode).processAsync("000", cloudClient);
    CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("000");
    boolean success = false;
    for (int i = 0; i < 200; i++) {
        CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
        if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
            success = true;
            break;
        }
        assertFalse(rsp.getRequestStatus() == RequestStatusState.FAILED);
        Thread.sleep(50);
    }
    assertTrue(success);
    try (HttpSolrClient coreclient = getHttpSolrClient(cloudClient.getZkStateReader().getBaseUrlForNodeName(node2bdecommissioned))) {
        CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient);
        assertTrue(status.getCoreStatus().size() == 0);
    }
    //let's do it back
    new CollectionAdminRequest.ReplaceNode(emptyNode, node2bdecommissioned).setParallel(Boolean.TRUE).processAsync("001", cloudClient);
    requestStatus = CollectionAdminRequest.requestStatus("001");
    for (int i = 0; i < 200; i++) {
        CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
        if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
            success = true;
            break;
        }
        assertFalse(rsp.getRequestStatus() == RequestStatusState.FAILED);
        Thread.sleep(50);
    }
    assertTrue(success);
    try (HttpSolrClient coreclient = getHttpSolrClient(cloudClient.getZkStateReader().getBaseUrlForNodeName(emptyNode))) {
        CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient);
        assertEquals("Expecting no cores but found some: " + status.getCoreStatus(), 0, status.getCoreStatus().size());
    }
    DocCollection collection = cloudClient.getZkStateReader().getClusterState().getCollection(coll);
    assertEquals(create.getNumShards().intValue(), collection.getSlices().size());
    for (Slice s : collection.getSlices()) {
        assertEquals(create.getNumNrtReplicas().intValue(), s.getReplicas(EnumSet.of(Replica.Type.NRT)).size());
        assertEquals(create.getNumTlogReplicas().intValue(), s.getReplicas(EnumSet.of(Replica.Type.TLOG)).size());
        assertEquals(create.getNumPullReplicas().intValue(), s.getReplicas(EnumSet.of(Replica.Type.PULL)).size());
    }
}

Example 50 with Slice

use of org.apache.solr.common.cloud.Slice in project lucene-solr by apache.

the class OverseerTest method testShardAssignmentBigger.

@Test
public void testShardAssignmentBigger() throws Exception {
    String zkDir = createTempDir("zkData").toFile().getAbsolutePath();
    //how many simulated nodes (num of threads)
    final int nodeCount = random().nextInt(TEST_NIGHTLY ? 50 : 10) + (TEST_NIGHTLY ? 50 : 10) + 1;
    //how many cores to register
    final int coreCount = random().nextInt(TEST_NIGHTLY ? 100 : 11) + (TEST_NIGHTLY ? 100 : 11) + 1;
    //how many slices
    final int sliceCount = random().nextInt(TEST_NIGHTLY ? 20 : 5) + 1;
    ZkTestServer server = new ZkTestServer(zkDir);
    SolrZkClient zkClient = null;
    ZkStateReader reader = null;
    SolrZkClient overseerClient = null;
    final MockZKController[] controllers = new MockZKController[nodeCount];
    final ExecutorService[] nodeExecutors = new ExecutorService[nodeCount];
    try {
        server.run();
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
        ZkController.createClusterZkNodes(zkClient);
        overseerClient = electNewOverseer(server.getZkAddress());
        reader = new ZkStateReader(zkClient);
        reader.createClusterStateWatchersAndUpdate();
        for (int i = 0; i < nodeCount; i++) {
            controllers[i] = new MockZKController(server.getZkAddress(), "node" + i);
        }
        for (int i = 0; i < nodeCount; i++) {
            nodeExecutors[i] = ExecutorUtil.newMDCAwareFixedThreadPool(1, new DefaultSolrThreadFactory("testShardAssignment"));
        }
        final String[] ids = new String[coreCount];
        //register total of coreCount cores
        for (int i = 0; i < coreCount; i++) {
            final int slot = i;
            nodeExecutors[i % nodeCount].submit((Runnable) () -> {
                final String coreName = "core" + slot;
                try {
                    ids[slot] = controllers[slot % nodeCount].publishState(COLLECTION, coreName, "node" + slot, Replica.State.ACTIVE, sliceCount);
                } catch (Throwable e) {
                    e.printStackTrace();
                    fail("register threw exception:" + e.getClass());
                }
            });
        }
        for (int i = 0; i < nodeCount; i++) {
            nodeExecutors[i].shutdown();
        }
        for (int i = 0; i < nodeCount; i++) {
            while (!nodeExecutors[i].awaitTermination(100, TimeUnit.MILLISECONDS)) ;
        }
        // make sure all cores have been assigned a id in cloudstate
        int cloudStateSliceCount = 0;
        for (int i = 0; i < 40; i++) {
            cloudStateSliceCount = 0;
            ClusterState state = reader.getClusterState();
            final Map<String, Slice> slices = state.getSlicesMap(COLLECTION);
            if (slices != null) {
                for (String name : slices.keySet()) {
                    cloudStateSliceCount += slices.get(name).getReplicasMap().size();
                }
                if (coreCount == cloudStateSliceCount)
                    break;
            }
            Thread.sleep(200);
        }
        assertEquals("Unable to verify all cores have been assigned an id in cloudstate", coreCount, cloudStateSliceCount);
        // make sure all cores have been returned an id
        int assignedCount = 0;
        for (int i = 0; i < 240; i++) {
            assignedCount = 0;
            for (int j = 0; j < coreCount; j++) {
                if (ids[j] != null) {
                    assignedCount++;
                }
            }
            if (coreCount == assignedCount) {
                break;
            }
            Thread.sleep(1000);
        }
        assertEquals("Unable to verify all cores have been returned an id", coreCount, assignedCount);
        final HashMap<String, AtomicInteger> counters = new HashMap<>();
        for (int i = 1; i < sliceCount + 1; i++) {
            counters.put("shard" + i, new AtomicInteger());
        }
        for (int i = 0; i < coreCount; i++) {
            final AtomicInteger ai = counters.get(ids[i]);
            assertNotNull("could not find counter for shard:" + ids[i], ai);
            ai.incrementAndGet();
        }
        for (String counter : counters.keySet()) {
            int count = counters.get(counter).intValue();
            int expectedCount = coreCount / sliceCount;
            int min = expectedCount - 1;
            int max = expectedCount + 1;
            if (count < min || count > max) {
                fail("Unevenly assigned shard ids, " + counter + " had " + count + ", expected: " + min + "-" + max);
            }
        }
        //make sure leaders are in cloud state
        for (int i = 0; i < sliceCount; i++) {
            assertNotNull(reader.getLeaderUrl(COLLECTION, "shard" + (i + 1), 15000));
        }
    } finally {
        close(zkClient);
        close(overseerClient);
        close(reader);
        for (int i = 0; i < controllers.length; i++) if (controllers[i] != null) {
            controllers[i].close();
        }
        server.shutdown();
        for (int i = 0; i < nodeCount; i++) {
            if (nodeExecutors[i] != null) {
                nodeExecutors[i].shutdownNow();
            }
        }
    }
}