Examples with DISTRIB_FROM org.apache.solr.update.processor.DistributedUpdateProcessor.DISTRIB_FROM used on opensource projects

Example 1 with DISTRIB_FROM

use of org.apache.solr.update.processor.DistributedUpdateProcessor.DISTRIB_FROM in project lucene-solr by apache.

the class DistributedVersionInfoTest method testReplicaVersionHandling.

@Test
public void testReplicaVersionHandling() throws Exception {
    final String shardId = "shard1";
    CollectionAdminRequest.createCollection(COLLECTION, "conf", 1, 3).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
    final ZkStateReader stateReader = cluster.getSolrClient().getZkStateReader();
    stateReader.waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS, (n, c) -> DocCollection.isFullyActive(n, c, 1, 3));
    final Replica leader = stateReader.getLeaderRetry(COLLECTION, shardId);
    // start by reloading the empty collection so we try to calculate the max from an empty index
    reloadCollection(leader, COLLECTION);
    sendDoc(1);
    cluster.getSolrClient().commit(COLLECTION);
    // verify doc is on the leader and replica
    final List<Replica> notLeaders = stateReader.getClusterState().getCollection(COLLECTION).getReplicas().stream().filter(r -> r.getCoreName().equals(leader.getCoreName()) == false).collect(Collectors.toList());
    assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, 1, null);
    // get max version from the leader and replica
    Replica replica = notLeaders.get(0);
    Long maxOnLeader = getMaxVersionFromIndex(leader);
    Long maxOnReplica = getMaxVersionFromIndex(replica);
    assertEquals("leader and replica should have same max version: " + maxOnLeader, maxOnLeader, maxOnReplica);
    // send the same doc but with a lower version than the max in the index
    try (SolrClient client = getHttpSolrClient(replica.getCoreUrl())) {
        String docId = String.valueOf(1);
        SolrInputDocument doc = new SolrInputDocument();
        doc.setField("id", docId);
        // bad version!!!
        doc.setField("_version_", maxOnReplica - 1);
        // simulate what the leader does when sending a doc to a replica
        ModifiableSolrParams params = new ModifiableSolrParams();
        params.set(DISTRIB_UPDATE_PARAM, DistributedUpdateProcessor.DistribPhase.FROMLEADER.toString());
        params.set(DISTRIB_FROM, leader.getCoreUrl());
        UpdateRequest req = new UpdateRequest();
        req.setParams(params);
        req.add(doc);
        log.info("Sending doc with out-of-date version (" + (maxOnReplica - 1) + ") document directly to replica");
        client.request(req);
        client.commit();
        Long docVersion = getVersionFromIndex(replica, docId);
        assertEquals("older version should have been thrown away", maxOnReplica, docVersion);
    }
    reloadCollection(leader, COLLECTION);
    maxOnLeader = getMaxVersionFromIndex(leader);
    maxOnReplica = getMaxVersionFromIndex(replica);
    assertEquals("leader and replica should have same max version after reload", maxOnLeader, maxOnReplica);
    // now start sending docs while collection is reloading
    delQ("*:*");
    commit();
    final Set<Integer> deletedDocs = new HashSet<>();
    final AtomicInteger docsSent = new AtomicInteger(0);
    final Random rand = new Random(5150);
    Thread docSenderThread = new Thread() {

        public void run() {
            // brief delay before sending docs
            try {
                Thread.sleep(rand.nextInt(30) + 1);
            } catch (InterruptedException e) {
            }
            for (int i = 0; i < 1000; i++) {
                if (i % (rand.nextInt(20) + 1) == 0) {
                    try {
                        Thread.sleep(rand.nextInt(50) + 1);
                    } catch (InterruptedException e) {
                    }
                }
                int docId = i + 1;
                try {
                    sendDoc(docId);
                    docsSent.incrementAndGet();
                } catch (Exception e) {
                }
            }
        }
    };
    Thread reloaderThread = new Thread() {

        public void run() {
            try {
                Thread.sleep(rand.nextInt(300) + 1);
            } catch (InterruptedException e) {
            }
            for (int i = 0; i < 3; i++) {
                try {
                    reloadCollection(leader, COLLECTION);
                } catch (Exception e) {
                }
                try {
                    Thread.sleep(rand.nextInt(300) + 300);
                } catch (InterruptedException e) {
                }
            }
        }
    };
    Thread deleteThread = new Thread() {

        public void run() {
            // brief delay before sending docs
            try {
                Thread.sleep(500);
            } catch (InterruptedException e) {
            }
            for (int i = 0; i < 200; i++) {
                try {
                    Thread.sleep(rand.nextInt(50) + 1);
                } catch (InterruptedException e) {
                }
                int ds = docsSent.get();
                if (ds > 0) {
                    int docToDelete = rand.nextInt(ds) + 1;
                    if (!deletedDocs.contains(docToDelete)) {
                        delI(String.valueOf(docToDelete));
                        deletedDocs.add(docToDelete);
                    }
                }
            }
        }
    };
    Thread committerThread = new Thread() {

        public void run() {
            try {
                Thread.sleep(rand.nextInt(200) + 1);
            } catch (InterruptedException e) {
            }
            for (int i = 0; i < 20; i++) {
                try {
                    cluster.getSolrClient().commit(COLLECTION);
                } catch (Exception e) {
                }
                try {
                    Thread.sleep(rand.nextInt(100) + 100);
                } catch (InterruptedException e) {
                }
            }
        }
    };
    docSenderThread.start();
    reloaderThread.start();
    committerThread.start();
    deleteThread.start();
    docSenderThread.join();
    reloaderThread.join();
    committerThread.join();
    deleteThread.join();
    cluster.getSolrClient().commit(COLLECTION);
    log.info("Total of " + deletedDocs.size() + " docs deleted");
    maxOnLeader = getMaxVersionFromIndex(leader);
    maxOnReplica = getMaxVersionFromIndex(replica);
    assertEquals("leader and replica should have same max version before reload", maxOnLeader, maxOnReplica);
    reloadCollection(leader, COLLECTION);
    maxOnLeader = getMaxVersionFromIndex(leader);
    maxOnReplica = getMaxVersionFromIndex(replica);
    assertEquals("leader and replica should have same max version after reload", maxOnLeader, maxOnReplica);
    assertDocsExistInAllReplicas(leader, notLeaders, COLLECTION, 1, 1000, deletedDocs);
}