Examples with SolrDocumentList org.apache.solr.common.SolrDocumentList used on opensource projects

Example 31 with SolrDocumentList

use of org.apache.solr.common.SolrDocumentList in project lucene-solr by apache.

the class TestReplicationHandler method testRateLimitedReplication.

@Test
public void testRateLimitedReplication() throws Exception {
    //clean index
    masterClient.deleteByQuery("*:*");
    slaveClient.deleteByQuery("*:*");
    masterClient.commit();
    slaveClient.commit();
    masterJetty.stop();
    slaveJetty.stop();
    //Start master with the new solrconfig
    master.copyConfigFile(CONF_DIR + "solrconfig-master-throttled.xml", "solrconfig.xml");
    useFactory(null);
    masterJetty = createJetty(master);
    masterClient.close();
    masterClient = createNewSolrClient(masterJetty.getLocalPort());
    //index docs
    final int totalDocs = TestUtil.nextInt(random(), 50, 100);
    for (int i = 0; i < totalDocs; i++) index(masterClient, "id", i, "name", TestUtil.randomSimpleString(random(), 1000, 5000));
    masterClient.commit();
    //Check Index Size
    String dataDir = master.getDataDir();
    masterClient.close();
    masterJetty.stop();
    Directory dir = FSDirectory.open(Paths.get(dataDir).resolve("index"));
    String[] files = dir.listAll();
    long totalBytes = 0;
    for (String file : files) {
        totalBytes += dir.fileLength(file);
    }
    // maxWriteMBPerSec=0.1 in solrconfig
    float approximateTimeInSeconds = Math.round(totalBytes / 1024 / 1024 / 0.1);
    //Start again and replicate the data
    useFactory(null);
    masterJetty = createJetty(master);
    masterClient = createNewSolrClient(masterJetty.getLocalPort());
    //start slave
    slave.setTestPort(masterJetty.getLocalPort());
    slave.copyConfigFile(CONF_DIR + "solrconfig-slave1.xml", "solrconfig.xml");
    slaveJetty = createJetty(slave);
    slaveClient.close();
    slaveClient = createNewSolrClient(slaveJetty.getLocalPort());
    long startTime = System.nanoTime();
    pullFromMasterToSlave();
    //Add a few more docs in the master. Just to make sure that we are replicating the correct index point
    //These extra docs should not get replicated
    new Thread(new AddExtraDocs(masterClient, totalDocs)).start();
    //Wait and make sure that it actually replicated correctly.
    NamedList slaveQueryRsp = rQuery(totalDocs, "*:*", slaveClient);
    SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
    assertEquals(totalDocs, slaveQueryResult.getNumFound());
    long timeTaken = System.nanoTime() - startTime;
    long timeTakenInSeconds = TimeUnit.SECONDS.convert(timeTaken, TimeUnit.NANOSECONDS);
    //Let's make sure it took more than approximateTimeInSeconds to make sure that it was throttled
    log.info("approximateTimeInSeconds = " + approximateTimeInSeconds + " timeTakenInSeconds = " + timeTakenInSeconds);
    assertTrue(timeTakenInSeconds - approximateTimeInSeconds > 0);
}

Example 32 with SolrDocumentList

use of org.apache.solr.common.SolrDocumentList in project lucene-solr by apache.

the class TestReplicationHandler method doTestReplicateAfterCoreReload.

@Test
public void doTestReplicateAfterCoreReload() throws Exception {
    int docs = TEST_NIGHTLY ? 200000 : 0;
    //stop slave
    slaveJetty.stop();
    //change solrconfig having 'replicateAfter startup' option on master
    master.copyConfigFile(CONF_DIR + "solrconfig-master3.xml", "solrconfig.xml");
    masterJetty.stop();
    masterJetty = createJetty(master);
    masterClient.close();
    masterClient = createNewSolrClient(masterJetty.getLocalPort());
    masterClient.deleteByQuery("*:*");
    for (int i = 0; i < docs; i++) index(masterClient, "id", i, "name", "name = " + i);
    masterClient.commit();
    NamedList masterQueryRsp = rQuery(docs, "*:*", masterClient);
    SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
    assertEquals(docs, masterQueryResult.getNumFound());
    slave.setTestPort(masterJetty.getLocalPort());
    slave.copyConfigFile(slave.getSolrConfigFile(), "solrconfig.xml");
    //start slave
    slaveJetty = createJetty(slave);
    slaveClient.close();
    slaveClient = createNewSolrClient(slaveJetty.getLocalPort());
    //get docs from slave and check if number is equal to master
    NamedList slaveQueryRsp = rQuery(docs, "*:*", slaveClient);
    SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
    assertEquals(docs, slaveQueryResult.getNumFound());
    //compare results
    String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
    assertEquals(null, cmp);
    Object version = getIndexVersion(masterClient).get("indexversion");
    NamedList<Object> commits = getCommits(masterClient);
    reloadCore(masterClient, "collection1");
    assertEquals(version, getIndexVersion(masterClient).get("indexversion"));
    assertEquals(commits.get("commits"), getCommits(masterClient).get("commits"));
    index(masterClient, "id", docs + 10, "name", "name = 1");
    index(masterClient, "id", docs + 20, "name", "name = 2");
    masterClient.commit();
    NamedList resp = rQuery(docs + 2, "*:*", masterClient);
    masterQueryResult = (SolrDocumentList) resp.get("response");
    assertEquals(docs + 2, masterQueryResult.getNumFound());
    //get docs from slave and check if number is equal to master
    slaveQueryRsp = rQuery(docs + 2, "*:*", slaveClient);
    slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
    assertEquals(docs + 2, slaveQueryResult.getNumFound());
}

Example 33 with SolrDocumentList

use of org.apache.solr.common.SolrDocumentList in project lucene-solr by apache.

the class TestReplicationHandler method doTestStressReplication.

@Test
public void doTestStressReplication() throws Exception {
    // change solrconfig on slave
    // this has no entry for pollinginterval
    // get us a straight standard fs dir rather than mock*dir
    boolean useStraightStandardDirectory = random().nextBoolean();
    if (useStraightStandardDirectory) {
        useFactory(null);
    }
    final String SLAVE_SCHEMA_1 = "schema-replication1.xml";
    final String SLAVE_SCHEMA_2 = "schema-replication2.xml";
    String slaveSchema = SLAVE_SCHEMA_1;
    try {
        slave.copyConfigFile(CONF_DIR + "solrconfig-slave1.xml", "solrconfig.xml");
        slave.copyConfigFile(CONF_DIR + slaveSchema, "schema.xml");
        slaveJetty.stop();
        slaveJetty = createJetty(slave);
        slaveClient.close();
        slaveClient = createNewSolrClient(slaveJetty.getLocalPort());
        master.copyConfigFile(CONF_DIR + "solrconfig-master3.xml", "solrconfig.xml");
        masterJetty.stop();
        masterJetty = createJetty(master);
        masterClient.close();
        masterClient = createNewSolrClient(masterJetty.getLocalPort());
        masterClient.deleteByQuery("*:*");
        slaveClient.deleteByQuery("*:*");
        slaveClient.commit();
        int maxDocs = TEST_NIGHTLY ? 1000 : 200;
        int rounds = TEST_NIGHTLY ? 80 : 8;
        int totalDocs = 0;
        int id = 0;
        for (int x = 0; x < rounds; x++) {
            final boolean confCoreReload = random().nextBoolean();
            if (confCoreReload) {
                // toggle the schema file used
                slaveSchema = slaveSchema.equals(SLAVE_SCHEMA_1) ? SLAVE_SCHEMA_2 : SLAVE_SCHEMA_1;
                master.copyConfigFile(CONF_DIR + slaveSchema, "schema.xml");
            }
            int docs = random().nextInt(maxDocs) + 1;
            for (int i = 0; i < docs; i++) {
                index(masterClient, "id", id++, "name", "name = " + i);
            }
            totalDocs += docs;
            masterClient.commit();
            NamedList masterQueryRsp = rQuery(totalDocs, "*:*", masterClient);
            SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
            assertEquals(totalDocs, masterQueryResult.getNumFound());
            // index fetch
            Date slaveCoreStart = watchCoreStartAt(slaveClient, 30 * 1000, null);
            pullFromMasterToSlave();
            if (confCoreReload) {
                watchCoreStartAt(slaveClient, 30 * 1000, slaveCoreStart);
            }
            // get docs from slave and check if number is equal to master
            NamedList slaveQueryRsp = rQuery(totalDocs, "*:*", slaveClient);
            SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
            assertEquals(totalDocs, slaveQueryResult.getNumFound());
            // compare results
            String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
            assertEquals(null, cmp);
            assertVersions(masterClient, slaveClient);
            checkForSingleIndex(masterJetty);
            checkForSingleIndex(slaveJetty);
            if (random().nextBoolean()) {
                // move the slave ahead
                for (int i = 0; i < 3; i++) {
                    index(slaveClient, "id", id++, "name", "name = " + i);
                }
                slaveClient.commit();
            }
        }
    } finally {
        if (useStraightStandardDirectory) {
            resetFactory();
        }
    }
}

Example 34 with SolrDocumentList

use of org.apache.solr.common.SolrDocumentList in project lucene-solr by apache.

the class KnnStream method open.

public void open() throws IOException {
    cloudSolrClient = cache.getCloudSolrClient(zkHost);
    ModifiableSolrParams params = getParams(this.props);
    StringBuilder builder = new StringBuilder();
    for (String key : mltParams) {
        if (params.get(key) != null) {
            builder.append(" " + key + "=" + params.get(key));
            params.remove(key);
        }
    }
    String k = params.get("k");
    if (k != null) {
        params.add(ROWS, k);
        params.remove(k);
    }
    params.add(Q, "{!mlt" + builder.toString() + "}" + id);
    QueryRequest request = new QueryRequest(params);
    try {
        QueryResponse response = request.process(cloudSolrClient, collection);
        SolrDocumentList docs = response.getResults();
        documentIterator = docs.iterator();
    } catch (Exception e) {
        throw new IOException(e);
    }
}

Example 35 with SolrDocumentList

use of org.apache.solr.common.SolrDocumentList in project lucene-solr by apache.

the class SolrTestCaseJ4 method compareSolrDocumentList.

public boolean compareSolrDocumentList(Object expected, Object actual) {
    if (!(expected instanceof SolrDocumentList) || !(actual instanceof SolrDocumentList)) {
        return false;
    }
    if (expected == actual) {
        return true;
    }
    SolrDocumentList list1 = (SolrDocumentList) expected;
    SolrDocumentList list2 = (SolrDocumentList) actual;
    if (list1.getMaxScore() == null) {
        if (list2.getMaxScore() != null) {
            return false;
        }
    } else if (list2.getMaxScore() == null) {
        return false;
    } else {
        if (Float.compare(list1.getMaxScore(), list2.getMaxScore()) != 0 || list1.getNumFound() != list2.getNumFound() || list1.getStart() != list2.getStart()) {
            return false;
        }
    }
    for (int i = 0; i < list1.getNumFound(); i++) {
        if (!compareSolrDocument(list1.get(i), list2.get(i))) {
            return false;
        }
    }
    return true;
}