Examples with ZkStateReader org.apache.solr.common.cloud.ZkStateReader used on opensource projects

Example 31 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class ZkStateReaderTest method testExternalCollectionWatchedNotWatched.

public void testExternalCollectionWatchedNotWatched() throws Exception {
    String zkDir = createTempDir("testExternalCollectionWatchedNotWatched").toFile().getAbsolutePath();
    ZkTestServer server = new ZkTestServer(zkDir);
    SolrZkClient zkClient = null;
    ZkStateReader reader = null;
    try {
        server.run();
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        zkClient = new SolrZkClient(server.getZkAddress(), OverseerTest.DEFAULT_CONNECTION_TIMEOUT);
        ZkController.createClusterZkNodes(zkClient);
        reader = new ZkStateReader(zkClient);
        reader.createClusterStateWatchersAndUpdate();
        ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats());
        zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true);
        // create new collection with stateFormat = 2
        ZkWriteCommand c1 = new ZkWriteCommand("c1", new DocCollection("c1", new HashMap<>(), new HashMap<>(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json"));
        writer.enqueueUpdate(reader.getClusterState(), c1, null);
        writer.writePendingUpdates();
        reader.forceUpdateCollection("c1");
        assertTrue(reader.getClusterState().getCollectionRef("c1").isLazilyLoaded());
        reader.registerCore("c1");
        assertFalse(reader.getClusterState().getCollectionRef("c1").isLazilyLoaded());
        reader.unregisterCore("c1");
        assertTrue(reader.getClusterState().getCollectionRef("c1").isLazilyLoaded());
    } finally {
        IOUtils.close(reader, zkClient);
        server.shutdown();
    }
}

Example 32 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class ZkStateWriterTest method testExternalModificationToStateFormat2.

public void testExternalModificationToStateFormat2() throws Exception {
    String zkDir = createTempDir("testExternalModificationToStateFormat2").toFile().getAbsolutePath();
    ZkTestServer server = new ZkTestServer(zkDir);
    SolrZkClient zkClient = null;
    try {
        server.run();
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        zkClient = new SolrZkClient(server.getZkAddress(), OverseerTest.DEFAULT_CONNECTION_TIMEOUT);
        ZkController.createClusterZkNodes(zkClient);
        try (ZkStateReader reader = new ZkStateReader(zkClient)) {
            reader.createClusterStateWatchersAndUpdate();
            ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats());
            zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true);
            zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c2", true);
            ClusterState state = reader.getClusterState();
            // create collection 2 with stateFormat = 2
            ZkWriteCommand c2 = new ZkWriteCommand("c2", new DocCollection("c2", new HashMap<String, Slice>(), new HashMap<String, Object>(), DocRouter.DEFAULT, 0, ZkStateReader.getCollectionPath("c2")));
            state = writer.enqueueUpdate(reader.getClusterState(), c2, null);
            // first write is flushed immediately
            assertFalse(writer.hasPendingUpdates());
            int sharedClusterStateVersion = state.getZkClusterStateVersion();
            int stateFormat2Version = state.getCollection("c2").getZNodeVersion();
            // Simulate an external modification to /collections/c2/state.json
            byte[] data = zkClient.getData(ZkStateReader.getCollectionPath("c2"), null, null, true);
            zkClient.setData(ZkStateReader.getCollectionPath("c2"), data, true);
            // get the most up-to-date state
            reader.forceUpdateCollection("c2");
            state = reader.getClusterState();
            log.info("Cluster state: {}", state);
            assertTrue(state.hasCollection("c2"));
            assertEquals(sharedClusterStateVersion, (int) state.getZkClusterStateVersion());
            assertEquals(stateFormat2Version + 1, state.getCollection("c2").getZNodeVersion());
            // enqueue an update to stateFormat2 collection such that update is pending
            state = writer.enqueueUpdate(state, c2, null);
            assertTrue(writer.hasPendingUpdates());
            // get the most up-to-date state
            reader.forceUpdateCollection("c2");
            state = reader.getClusterState();
            // enqueue a stateFormat=1 collection which should cause a flush
            ZkWriteCommand c1 = new ZkWriteCommand("c1", new DocCollection("c1", new HashMap<String, Slice>(), new HashMap<String, Object>(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE));
            try {
                writer.enqueueUpdate(state, c1, null);
                fail("Enqueue should not have succeeded");
            } catch (KeeperException.BadVersionException bve) {
            // expected
            }
            try {
                writer.enqueueUpdate(reader.getClusterState(), c2, null);
                fail("enqueueUpdate after BadVersionException should not have succeeded");
            } catch (IllegalStateException e) {
            // expected
            }
            try {
                writer.writePendingUpdates();
                fail("writePendingUpdates after BadVersionException should not have succeeded");
            } catch (IllegalStateException e) {
            // expected
            }
        }
    } finally {
        IOUtils.close(zkClient);
        server.shutdown();
    }
}

Example 33 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class ZkStateWriterTest method testSingleExternalCollection.

public void testSingleExternalCollection() throws Exception {
    String zkDir = createTempDir("testSingleExternalCollection").toFile().getAbsolutePath();
    ZkTestServer server = new ZkTestServer(zkDir);
    SolrZkClient zkClient = null;
    try {
        server.run();
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        zkClient = new SolrZkClient(server.getZkAddress(), OverseerTest.DEFAULT_CONNECTION_TIMEOUT);
        ZkController.createClusterZkNodes(zkClient);
        try (ZkStateReader reader = new ZkStateReader(zkClient)) {
            reader.createClusterStateWatchersAndUpdate();
            ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats());
            zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true);
            // create new collection with stateFormat = 2
            ZkWriteCommand c1 = new ZkWriteCommand("c1", new DocCollection("c1", new HashMap<String, Slice>(), new HashMap<String, Object>(), DocRouter.DEFAULT, 0, ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json"));
            ClusterState clusterState = writer.enqueueUpdate(reader.getClusterState(), c1, null);
            writer.writePendingUpdates();
            Map map = (Map) Utils.fromJSON(zkClient.getData("/clusterstate.json", null, null, true));
            assertNull(map.get("c1"));
            map = (Map) Utils.fromJSON(zkClient.getData(ZkStateReader.COLLECTIONS_ZKNODE + "/c1/state.json", null, null, true));
            assertNotNull(map.get("c1"));
        }
    } finally {
        IOUtils.close(zkClient);
        server.shutdown();
    }
}

Example 34 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class ZkStateWriterTest method testExternalModificationToSharedClusterState.

public void testExternalModificationToSharedClusterState() throws Exception {
    String zkDir = createTempDir("testExternalModification").toFile().getAbsolutePath();
    ZkTestServer server = new ZkTestServer(zkDir);
    SolrZkClient zkClient = null;
    try {
        server.run();
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        zkClient = new SolrZkClient(server.getZkAddress(), OverseerTest.DEFAULT_CONNECTION_TIMEOUT);
        ZkController.createClusterZkNodes(zkClient);
        try (ZkStateReader reader = new ZkStateReader(zkClient)) {
            reader.createClusterStateWatchersAndUpdate();
            ZkStateWriter writer = new ZkStateWriter(reader, new Overseer.Stats());
            zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c1", true);
            zkClient.makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/c2", true);
            // create collection 1 with stateFormat = 1
            ZkWriteCommand c1 = new ZkWriteCommand("c1", new DocCollection("c1", new HashMap<String, Slice>(), new HashMap<String, Object>(), DocRouter.DEFAULT, 0, ZkStateReader.CLUSTER_STATE));
            writer.enqueueUpdate(reader.getClusterState(), c1, null);
            writer.writePendingUpdates();
            reader.forceUpdateCollection("c1");
            reader.forceUpdateCollection("c2");
            // keep a reference to the current cluster state object
            ClusterState clusterState = reader.getClusterState();
            assertTrue(clusterState.hasCollection("c1"));
            assertFalse(clusterState.hasCollection("c2"));
            // Simulate an external modification to /clusterstate.json
            byte[] data = zkClient.getData("/clusterstate.json", null, null, true);
            zkClient.setData("/clusterstate.json", data, true);
            // enqueue another c1 so that ZkStateWriter has pending updates
            writer.enqueueUpdate(clusterState, c1, null);
            assertTrue(writer.hasPendingUpdates());
            // create collection 2 with stateFormat = 1
            ZkWriteCommand c2 = new ZkWriteCommand("c2", new DocCollection("c2", new HashMap<String, Slice>(), new HashMap<String, Object>(), DocRouter.DEFAULT, 0, ZkStateReader.getCollectionPath("c2")));
            try {
                // we are sending in the old cluster state object
                writer.enqueueUpdate(clusterState, c2, null);
                fail("Enqueue should not have succeeded");
            } catch (KeeperException.BadVersionException bve) {
            // expected
            }
            try {
                writer.enqueueUpdate(reader.getClusterState(), c2, null);
                fail("enqueueUpdate after BadVersionException should not have succeeded");
            } catch (IllegalStateException e) {
            // expected
            }
            try {
                writer.writePendingUpdates();
                fail("writePendingUpdates after BadVersionException should not have succeeded");
            } catch (IllegalStateException e) {
            // expected
            }
        }
    } finally {
        IOUtils.close(zkClient);
        server.shutdown();
    }
}

Example 35 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class AbstractFullDistribZkTestBase method ensureAllReplicasAreActive.

protected List<Replica> ensureAllReplicasAreActive(String testCollectionName, String shardId, int shards, int rf, int maxWaitSecs) throws Exception {
    final RTimer timer = new RTimer();
    Map<String, Replica> notLeaders = new HashMap<>();
    ZkStateReader zkr = cloudClient.getZkStateReader();
    // force the state to be fresh
    zkr.forceUpdateCollection(testCollectionName);
    ClusterState cs = zkr.getClusterState();
    Collection<Slice> slices = cs.getActiveSlices(testCollectionName);
    assertTrue(slices.size() == shards);
    boolean allReplicasUp = false;
    long waitMs = 0L;
    long maxWaitMs = maxWaitSecs * 1000L;
    Replica leader = null;
    while (waitMs < maxWaitMs && !allReplicasUp) {
        cs = cloudClient.getZkStateReader().getClusterState();
        assertNotNull(cs);
        Slice shard = cs.getSlice(testCollectionName, shardId);
        assertNotNull("No Slice for " + shardId, shard);
        // assume true
        allReplicasUp = true;
        Collection<Replica> replicas = shard.getReplicas();
        assertTrue("Did not find correct number of replicas. Expected:" + rf + " Found:" + replicas.size(), replicas.size() == rf);
        leader = shard.getLeader();
        assertNotNull(leader);
        log.info("Found " + replicas.size() + " replicas and leader on " + leader.getNodeName() + " for " + shardId + " in " + testCollectionName);
        // ensure all replicas are "active" and identify the non-leader replica
        for (Replica replica : replicas) {
            if (replica.getState() != Replica.State.ACTIVE) {
                log.info("Replica {} is currently {}", replica.getName(), replica.getState());
                allReplicasUp = false;
            }
            if (!leader.equals(replica))
                notLeaders.put(replica.getName(), replica);
        }
        if (!allReplicasUp) {
            try {
                Thread.sleep(500L);
            } catch (Exception ignoreMe) {
            }
            waitMs += 500L;
        }
    }
    if (!allReplicasUp)
        fail("Didn't see all replicas for shard " + shardId + " in " + testCollectionName + " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo());
    if (notLeaders.isEmpty())
        fail("Didn't isolate any replicas that are not the leader! ClusterState: " + printClusterStateInfo());
    log.info("Took {} ms to see all replicas become active.", timer.getTime());
    List<Replica> replicas = new ArrayList<>();
    replicas.addAll(notLeaders.values());
    return replicas;
}