Examples with SolrZkClient org.apache.solr.common.cloud.SolrZkClient used on opensource projects

Example 96 with SolrZkClient

use of org.apache.solr.common.cloud.SolrZkClient in project lucene-solr by apache.

the class FullSolrCloudDistribCmdsTest method testThatCantForwardToLeaderFails.

private long testThatCantForwardToLeaderFails(long docId) throws Exception {
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ZkNodeProps props = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard1");
    chaosMonkey.stopShard("shard1");
    Thread.sleep(1000);
    // fake that the leader is still advertised
    String leaderPath = ZkStateReader.getShardLeadersPath(DEFAULT_COLLECTION, "shard1");
    SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), 10000);
    int fails = 0;
    try {
        zkClient.makePath(leaderPath, Utils.toJSON(props), CreateMode.EPHEMERAL, true);
        for (int i = 0; i < 200; i++) {
            try {
                index_specific(shardToJetty.get("shard2").get(0).client.solrClient, id, docId++);
            } catch (SolrException e) {
                // expected
                fails++;
                break;
            } catch (SolrServerException e) {
                // expected
                fails++;
                break;
            }
        }
    } finally {
        zkClient.close();
    }
    assertTrue("A whole shard is down - some of these should fail", fails > 0);
    return docId;
}

Example 97 with SolrZkClient

use of org.apache.solr.common.cloud.SolrZkClient in project lucene-solr by apache.

the class OverseerTest method testExternalClusterStateChangeBehavior.

@Test
public void testExternalClusterStateChangeBehavior() throws Exception {
    String zkDir = createTempDir("testExternalClusterStateChangeBehavior").toFile().getAbsolutePath();
    ZkTestServer server = new ZkTestServer(zkDir);
    SolrZkClient zkClient = null;
    ZkStateReader reader = null;
    SolrZkClient overseerClient = null;
    try {
        server.run();
        zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        ZkController.createClusterZkNodes(zkClient);
        zkClient.create("/collections/test", null, CreateMode.PERSISTENT, true);
        reader = new ZkStateReader(zkClient);
        reader.createClusterStateWatchersAndUpdate();
        overseerClient = electNewOverseer(server.getZkAddress());
        DistributedQueue q = Overseer.getStateUpdateQueue(zkClient);
        ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", ZkStateReader.NODE_NAME_PROP, "node1", ZkStateReader.COLLECTION_PROP, "c1", ZkStateReader.CORE_NAME_PROP, "core1", ZkStateReader.ROLES_PROP, "", ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
        q.offer(Utils.toJSON(m));
        waitForCollections(reader, "c1");
        verifyReplicaStatus(reader, "c1", "shard1", "core_node1", Replica.State.DOWN);
        m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", ZkStateReader.NODE_NAME_PROP, "node1", ZkStateReader.COLLECTION_PROP, "c1", ZkStateReader.CORE_NAME_PROP, "core1", ZkStateReader.ROLES_PROP, "", ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString());
        q.offer(Utils.toJSON(m));
        m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", ZkStateReader.NODE_NAME_PROP, "node1", ZkStateReader.COLLECTION_PROP, "c1", ZkStateReader.CORE_NAME_PROP, "core1", ZkStateReader.ROLES_PROP, "", ZkStateReader.STATE_PROP, Replica.State.ACTIVE.toString());
        q.offer(Utils.toJSON(m));
        Stat stat = new Stat();
        byte[] data = zkClient.getData("/clusterstate.json", null, stat, true);
        // Simulate an external modification
        zkClient.setData("/clusterstate.json", data, true);
        m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), "name", "test", ZkStateReader.NUM_SHARDS_PROP, "1", ZkStateReader.REPLICATION_FACTOR, "1", DocCollection.STATE_FORMAT, "2");
        q.offer(Utils.toJSON(m));
        m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATESHARD.toLower(), "collection", "test", ZkStateReader.SHARD_ID_PROP, "x", ZkStateReader.REPLICATION_FACTOR, "1");
        q.offer(Utils.toJSON(m));
        m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.ADDREPLICA.toLower(), "collection", "test", ZkStateReader.SHARD_ID_PROP, "x", ZkStateReader.BASE_URL_PROP, "http://127.0.0.1/solr", ZkStateReader.NODE_NAME_PROP, "node1", ZkStateReader.CORE_NAME_PROP, "core1", ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
        q.offer(Utils.toJSON(m));
        waitForCollections(reader, "test");
        verifyReplicaStatus(reader, "test", "x", "core_node1", Replica.State.DOWN);
        waitForCollections(reader, "c1");
        verifyReplicaStatus(reader, "c1", "shard1", "core_node1", Replica.State.ACTIVE);
    } finally {
        close(zkClient);
        close(overseerClient);
        close(reader);
        server.shutdown();
    }
}

Example 98 with SolrZkClient

use of org.apache.solr.common.cloud.SolrZkClient in project lucene-solr by apache.

the class OverseerTest method testDoubleAssignment.

@Test
public void testDoubleAssignment() throws Exception {
    String zkDir = createTempDir("zkData").toFile().getAbsolutePath();
    ZkTestServer server = new ZkTestServer(zkDir);
    SolrZkClient controllerClient = null;
    SolrZkClient overseerClient = null;
    ZkStateReader reader = null;
    MockZKController mockController = null;
    try {
        server.run();
        controllerClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        ZkController.createClusterZkNodes(controllerClient);
        reader = new ZkStateReader(controllerClient);
        reader.createClusterStateWatchersAndUpdate();
        mockController = new MockZKController(server.getZkAddress(), "node1");
        overseerClient = electNewOverseer(server.getZkAddress());
        mockController.publishState(COLLECTION, "core1", "core_node1", Replica.State.RECOVERING, 1);
        waitForCollections(reader, "collection1");
        verifyReplicaStatus(reader, COLLECTION, "shard1", "core_node1", Replica.State.RECOVERING);
        mockController.close();
        int version = getClusterStateVersion(controllerClient);
        mockController = new MockZKController(server.getZkAddress(), "node1");
        mockController.publishState(COLLECTION, "core1", "core_node1", Replica.State.RECOVERING, 1);
        while (version == reader.getClusterState().getZkClusterStateVersion()) {
            Thread.sleep(100);
        }
        ClusterState state = reader.getClusterState();
        int numFound = 0;
        Map<String, DocCollection> collectionsMap = state.getCollectionsMap();
        for (Map.Entry<String, DocCollection> entry : collectionsMap.entrySet()) {
            DocCollection collection = entry.getValue();
            for (Slice slice : collection.getSlices()) {
                if (slice.getReplicasMap().get("core_node1") != null) {
                    numFound++;
                }
            }
        }
        assertEquals("Shard was found more than once in ClusterState", 1, numFound);
    } finally {
        close(overseerClient);
        close(mockController);
        close(controllerClient);
        close(reader);
        server.shutdown();
    }
}

Example 99 with SolrZkClient

use of org.apache.solr.common.cloud.SolrZkClient in project lucene-solr by apache.

the class OverseerTest method testBadQueueItem.

@Test
public void testBadQueueItem() throws Exception {
    String zkDir = createTempDir("zkData").toFile().getAbsolutePath();
    ZkTestServer server = new ZkTestServer(zkDir);
    MockZKController zkController = null;
    SolrZkClient zkClient = null;
    SolrZkClient overseerClient = null;
    try {
        server.run();
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
        ZkController.createClusterZkNodes(zkClient);
        overseerClient = electNewOverseer(server.getZkAddress());
        ZkStateReader reader = new ZkStateReader(zkClient);
        reader.createClusterStateWatchersAndUpdate();
        zkController = new MockZKController(server.getZkAddress(), "127.0.0.1");
        final int numShards = 3;
        for (int i = 0; i < numShards; i++) {
            assertNotNull("shard got no id?", zkController.publishState(COLLECTION, "core" + (i + 1), "node" + (i + 1), Replica.State.ACTIVE, 3));
        }
        assertEquals(1, reader.getClusterState().getSlice(COLLECTION, "shard1").getReplicasMap().size());
        assertEquals(1, reader.getClusterState().getSlice(COLLECTION, "shard2").getReplicasMap().size());
        assertEquals(1, reader.getClusterState().getSlice(COLLECTION, "shard3").getReplicasMap().size());
        //make sure leaders are in cloud state
        assertNotNull(reader.getLeaderUrl(COLLECTION, "shard1", 15000));
        assertNotNull(reader.getLeaderUrl(COLLECTION, "shard2", 15000));
        assertNotNull(reader.getLeaderUrl(COLLECTION, "shard3", 15000));
        // publish a bad queue item
        String emptyCollectionName = "";
        zkController.publishState(emptyCollectionName, "core0", "node0", Replica.State.ACTIVE, 1);
        zkController.publishState(emptyCollectionName, "core0", "node0", null, 1);
        // make sure the Overseer is still processing items
        for (int i = 0; i < numShards; i++) {
            assertNotNull("shard got no id?", zkController.publishState("collection2", "core" + (i + 1), "node" + (i + 1), Replica.State.ACTIVE, 3));
        }
        assertEquals(1, reader.getClusterState().getSlice("collection2", "shard1").getReplicasMap().size());
        assertEquals(1, reader.getClusterState().getSlice("collection2", "shard2").getReplicasMap().size());
        assertEquals(1, reader.getClusterState().getSlice("collection2", "shard3").getReplicasMap().size());
        //make sure leaders are in cloud state
        assertNotNull(reader.getLeaderUrl("collection2", "shard1", 15000));
        assertNotNull(reader.getLeaderUrl("collection2", "shard2", 15000));
        assertNotNull(reader.getLeaderUrl("collection2", "shard3", 15000));
    } finally {
        close(zkClient);
        if (zkController != null) {
            zkController.close();
        }
        close(overseerClient);
        server.shutdown();
    }
}

Example 100 with SolrZkClient

use of org.apache.solr.common.cloud.SolrZkClient in project lucene-solr by apache.

the class OverseerTest method testPerformance.

@Test
@Ignore
public void testPerformance() throws Exception {
    String zkDir = createTempDir("OverseerTest.testPerformance").toFile().getAbsolutePath();
    ZkTestServer server = new ZkTestServer(zkDir);
    SolrZkClient controllerClient = null;
    SolrZkClient overseerClient = null;
    ZkStateReader reader = null;
    MockZKController mockController = null;
    try {
        server.run();
        controllerClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        ZkController.createClusterZkNodes(controllerClient);
        reader = new ZkStateReader(controllerClient);
        reader.createClusterStateWatchersAndUpdate();
        mockController = new MockZKController(server.getZkAddress(), "node1");
        final int MAX_COLLECTIONS = 10, MAX_CORES = 10, MAX_STATE_CHANGES = 20000, STATE_FORMAT = 2;
        for (int i = 0; i < MAX_COLLECTIONS; i++) {
            ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.CREATE.toLower(), "name", "perf" + i, ZkStateReader.NUM_SHARDS_PROP, "1", "stateFormat", String.valueOf(STATE_FORMAT), ZkStateReader.REPLICATION_FACTOR, "1", ZkStateReader.MAX_SHARDS_PER_NODE, "1");
            DistributedQueue q = Overseer.getStateUpdateQueue(controllerClient);
            q.offer(Utils.toJSON(m));
            controllerClient.makePath("/collections/perf" + i, true);
        }
        for (int i = 0, j = 0, k = 0; i < MAX_STATE_CHANGES; i++, j++, k++) {
            ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString(), ZkStateReader.NODE_NAME_PROP, "node1", ZkStateReader.CORE_NAME_PROP, "core" + k, ZkStateReader.CORE_NODE_NAME_PROP, "node1", ZkStateReader.COLLECTION_PROP, "perf" + j, ZkStateReader.NUM_SHARDS_PROP, "1", ZkStateReader.BASE_URL_PROP, "http://" + "node1" + "/solr/");
            DistributedQueue q = Overseer.getStateUpdateQueue(controllerClient);
            q.offer(Utils.toJSON(m));
            if (j >= MAX_COLLECTIONS - 1)
                j = 0;
            if (k >= MAX_CORES - 1)
                k = 0;
            if (i > 0 && i % 100 == 0)
                log.info("Published {} items", i);
        }
        // let's publish a sentinel collection which we'll use to wait for overseer to complete operations
        ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(), ZkStateReader.STATE_PROP, Replica.State.ACTIVE.toString(), ZkStateReader.NODE_NAME_PROP, "node1", ZkStateReader.CORE_NAME_PROP, "core1", ZkStateReader.CORE_NODE_NAME_PROP, "node1", ZkStateReader.COLLECTION_PROP, "perf_sentinel", ZkStateReader.NUM_SHARDS_PROP, "1", ZkStateReader.BASE_URL_PROP, "http://" + "node1" + "/solr/");
        DistributedQueue q = Overseer.getStateUpdateQueue(controllerClient);
        q.offer(Utils.toJSON(m));
        Timer t = new Timer();
        Timer.Context context = t.time();
        try {
            overseerClient = electNewOverseer(server.getZkAddress());
            assertTrue(overseers.size() > 0);
            while (true) {
                ClusterState state = reader.getClusterState();
                if (state.hasCollection("perf_sentinel")) {
                    break;
                }
                Thread.sleep(1000);
            }
        } finally {
            context.stop();
        }
        log.info("Overseer loop finished processing: ");
        printTimingStats(t);
        Overseer overseer = overseers.get(0);
        Overseer.Stats stats = overseer.getStats();
        String[] interestingOps = { "state", "update_state", "am_i_leader", "" };
        Arrays.sort(interestingOps);
        for (Map.Entry<String, Overseer.Stat> entry : stats.getStats().entrySet()) {
            String op = entry.getKey();
            if (Arrays.binarySearch(interestingOps, op) < 0)
                continue;
            Overseer.Stat stat = entry.getValue();
            log.info("op: {}, success: {}, failure: {}", op, stat.success.get(), stat.errors.get());
            Timer timer = stat.requestTime;
            printTimingStats(timer);
        }
    } finally {
        close(overseerClient);
        close(mockController);
        close(controllerClient);
        close(reader);
        server.shutdown();
    }
}