Examples with ZkStateReader org.apache.solr.common.cloud.ZkStateReader used on opensource projects

Example 1 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project titan by thinkaurelius.

the class SolrIndex method waitForRecoveriesToFinish.

/**
     * Wait for all the collection shards to be ready.
     */
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    try {
        boolean cont = true;
        while (cont) {
            boolean sawLiveRecovering = false;
            zkStateReader.updateClusterState(true);
            ClusterState clusterState = zkStateReader.getClusterState();
            Map<String, Slice> slices = clusterState.getSlicesMap(collection);
            Preconditions.checkNotNull("Could not find collection:" + collection, slices);
            // remove SYNC state per: http://tinyurl.com/pag6rwt
            for (Map.Entry<String, Slice> entry : slices.entrySet()) {
                Map<String, Replica> shards = entry.getValue().getReplicasMap();
                for (Map.Entry<String, Replica> shard : shards.entrySet()) {
                    String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
                    if ((state.equals(Replica.State.RECOVERING) || state.equals(Replica.State.DOWN)) && clusterState.liveNodesContain(shard.getValue().getStr(ZkStateReader.NODE_NAME_PROP))) {
                        sawLiveRecovering = true;
                    }
                }
            }
            if (!sawLiveRecovering) {
                cont = false;
            } else {
                Thread.sleep(1000);
            }
        }
    } finally {
        logger.info("Exiting solr wait");
    }
}

Example 2 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project incubator-atlas by apache.

the class Solr5Index method clearStorage.

@Override
public void clearStorage() throws BackendException {
    try {
        if (mode != Mode.CLOUD)
            throw new UnsupportedOperationException("Operation only supported for SolrCloud");
        logger.debug("Clearing storage from Solr: {}", solrClient);
        ZkStateReader zkStateReader = ((CloudSolrClient) solrClient).getZkStateReader();
        zkStateReader.updateClusterState();
        ClusterState clusterState = zkStateReader.getClusterState();
        for (String collection : clusterState.getCollections()) {
            logger.debug("Clearing collection [{}] in Solr", collection);
            UpdateRequest deleteAll = newUpdateRequest();
            deleteAll.deleteByQuery("*:*");
            solrClient.request(deleteAll, collection);
        }
    } catch (SolrServerException e) {
        logger.error("Unable to clear storage from index due to server error on Solr.", e);
        throw new PermanentBackendException(e);
    } catch (IOException e) {
        logger.error("Unable to clear storage from index due to low-level I/O error.", e);
        throw new PermanentBackendException(e);
    } catch (Exception e) {
        logger.error("Unable to clear storage from index due to general error.", e);
        throw new PermanentBackendException(e);
    }
}

Example 3 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project incubator-atlas by apache.

the class Solr5Index method checkIfCollectionExists.

/**
     * Checks if the collection has already been created in Solr.
     */
private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    zkStateReader.updateClusterState();
    ClusterState clusterState = zkStateReader.getClusterState();
    return clusterState.getCollectionOrNull(collection) != null;
}

Example 4 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project incubator-atlas by apache.

the class Solr5Index method waitForRecoveriesToFinish.

/**
     * Wait for all the collection shards to be ready.
     */
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    try {
        boolean cont = true;
        while (cont) {
            boolean sawLiveRecovering = false;
            zkStateReader.updateClusterState();
            ClusterState clusterState = zkStateReader.getClusterState();
            Map<String, Slice> slices = clusterState.getSlicesMap(collection);
            Preconditions.checkNotNull("Could not find collection:" + collection, slices);
            for (Map.Entry<String, Slice> entry : slices.entrySet()) {
                Map<String, Replica> shards = entry.getValue().getReplicasMap();
                for (Map.Entry<String, Replica> shard : shards.entrySet()) {
                    String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
                    if ((state.equals(Replica.State.RECOVERING.toString()) || state.equals(Replica.State.DOWN.toString())) && clusterState.liveNodesContain(shard.getValue().getStr(ZkStateReader.NODE_NAME_PROP))) {
                        sawLiveRecovering = true;
                    }
                }
            }
            if (!sawLiveRecovering) {
                cont = false;
            } else {
                Thread.sleep(1000);
            }
        }
    } finally {
        logger.info("Exiting solr wait");
    }
}

Example 5 with ZkStateReader

use of org.apache.solr.common.cloud.ZkStateReader in project lucene-solr by apache.

the class DistributedVersionInfoTest method testReplicaVersionHandling.

@Test
public void testReplicaVersionHandling() throws Exception {
    final String shardId = "shard1";
    CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 3).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
    final ZkStateReader stateReader = cluster.getSolrClient().getZkStateReader();
    stateReader.waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS, (n, c) -> DocCollection.isFullyActive(n, c, 1, 3));
    final Replica leader = stateReader.getLeaderRetry(COLLECTION, shardId);
    // start by reloading the empty collection so we try to calculate the max from an empty index
    reloadCollection(leader, COLLECTION);
    sendDoc(1);
    cluster.getSolrClient().commit(COLLECTION);
    // verify doc is on the leader and replica
    final List<Replica> notLeaders = stateReader.getClusterState().getCollection(COLLECTION).getReplicas().stream().filter(r -> r.getCoreName().equals(leader.getCoreName()) == false).collect(Collectors.toList());
    assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, 1, null);
    // get max version from the leader and replica
    Replica replica = notLeaders.get(0);
    Long maxOnLeader = getMaxVersionFromIndex(leader);
    Long maxOnReplica = getMaxVersionFromIndex(replica);
    assertEquals("leader and replica should have same max version: " + maxOnLeader, maxOnLeader, maxOnReplica);
    // send the same doc but with a lower version than the max in the index
    try (SolrClient client = getHttpSolrClient(replica.getCoreUrl())) {
        String docId = String.valueOf(1);
        SolrInputDocument doc = new SolrInputDocument();
        doc.setField("id", docId);
        // bad version!!!
        doc.setField("_version_", maxOnReplica - 1);
        // simulate what the leader does when sending a doc to a replica
        ModifiableSolrParams params = new ModifiableSolrParams();
        params.set(DISTRIB_UPDATE_PARAM, DistributedUpdateProcessor.DistribPhase.FROMLEADER.toString());
        params.set(DISTRIB_FROM, leader.getCoreUrl());
        UpdateRequest req = new UpdateRequest();
        req.setParams(params);
        req.add(doc);
        log.info("Sending doc with out-of-date version (" + (maxOnReplica - 1) + ") document directly to replica");
        client.request(req);
        client.commit();
        Long docVersion = getVersionFromIndex(replica, docId);
        assertEquals("older version should have been thrown away", maxOnReplica, docVersion);
    }
    reloadCollection(leader, COLLECTION);
    maxOnLeader = getMaxVersionFromIndex(leader);
    maxOnReplica = getMaxVersionFromIndex(replica);
    assertEquals("leader and replica should have same max version after reload", maxOnLeader, maxOnReplica);
    // now start sending docs while collection is reloading
    delQ("*:*");
    commit();
    final Set<Integer> deletedDocs = new HashSet<>();
    final AtomicInteger docsSent = new AtomicInteger(0);
    final Random rand = new Random(5150);
    Thread docSenderThread = new Thread() {

        public void run() {
            // brief delay before sending docs
            try {
                Thread.sleep(rand.nextInt(30) + 1);
            } catch (InterruptedException e) {
            }
            for (int i = 0; i < 1000; i++) {
                if (i % (rand.nextInt(20) + 1) == 0) {
                    try {
                        Thread.sleep(rand.nextInt(50) + 1);
                    } catch (InterruptedException e) {
                    }
                }
                int docId = i + 1;
                try {
                    sendDoc(docId);
                    docsSent.incrementAndGet();
                } catch (Exception e) {
                }
            }
        }
    };
    Thread reloaderThread = new Thread() {

        public void run() {
            try {
                Thread.sleep(rand.nextInt(300) + 1);
            } catch (InterruptedException e) {
            }
            for (int i = 0; i < 3; i++) {
                try {
                    reloadCollection(leader, COLLECTION);
                } catch (Exception e) {
                }
                try {
                    Thread.sleep(rand.nextInt(300) + 300);
                } catch (InterruptedException e) {
                }
            }
        }
    };
    Thread deleteThread = new Thread() {

        public void run() {
            // brief delay before sending docs
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
            }
            for (int i = 0; i < 200; i++) {
                try {
                    Thread.sleep(rand.nextInt(50) + 1);
                } catch (InterruptedException e) {
                }
                int ds = docsSent.get();
                if (ds > 0) {
                    int docToDelete = rand.nextInt(ds) + 1;
                    if (!deletedDocs.contains(docToDelete)) {
                        delI(String.valueOf(docToDelete));
                        deletedDocs.add(docToDelete);
                    }
                }
            }
        }
    };
    Thread committerThread = new Thread() {

        public void run() {
            try {
                Thread.sleep(rand.nextInt(200) + 1);
            } catch (InterruptedException e) {
            }
            for (int i = 0; i < 20; i++) {
                try {
                    cluster.getSolrClient().commit(COLLECTION);
                } catch (Exception e) {
                }
                try {
                    Thread.sleep(rand.nextInt(100) + 100);
                } catch (InterruptedException e) {
                }
            }
        }
    };
    docSenderThread.start();
    reloaderThread.start();
    committerThread.start();
    deleteThread.start();
    docSenderThread.join();
    reloaderThread.join();
    committerThread.join();
    deleteThread.join();
    cluster.getSolrClient().commit(COLLECTION);
    log.info("Total of " + deletedDocs.size() + " docs deleted");
    maxOnLeader = getMaxVersionFromIndex(leader);
    maxOnReplica = getMaxVersionFromIndex(replica);
    assertEquals("leader and replica should have same max version before reload", maxOnLeader, maxOnReplica);
    reloadCollection(leader, COLLECTION);
    maxOnLeader = getMaxVersionFromIndex(leader);
    maxOnReplica = getMaxVersionFromIndex(replica);
    assertEquals("leader and replica should have same max version after reload", maxOnLeader, maxOnReplica);
    assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, 1000, deletedDocs);
}