Examples with SolrInputDocument org.apache.solr.common.SolrInputDocument used on opensource projects

Example 46 with SolrInputDocument

use of org.apache.solr.common.SolrInputDocument in project lucene-solr by apache.

the class DocumentBuilderTest method testExceptions.

@Test
public void testExceptions() {
    SolrCore core = h.getCore();
    // make sure a null value is not indexed
    SolrInputDocument doc = new SolrInputDocument();
    doc.addField("id", "123");
    doc.addField("unknown", "something");
    try {
        DocumentBuilder.toDocument(doc, core.getLatestSchema());
        fail("added an unknown field");
    } catch (Exception ex) {
        assertTrue("should have document ID", ex.getMessage().indexOf("doc=123") > 0);
    }
    doc.remove("unknown");
    doc.addField("weight", "not a number");
    try {
        DocumentBuilder.toDocument(doc, core.getLatestSchema());
        fail("invalid 'float' field value");
    } catch (Exception ex) {
        assertTrue("should have document ID", ex.getMessage().indexOf("doc=123") > 0);
        assertTrue("cause is number format", ex.getCause() instanceof NumberFormatException);
    }
    // now make sure it is OK
    doc.setField("weight", "1.34");
    DocumentBuilder.toDocument(doc, core.getLatestSchema());
}

Example 47 with SolrInputDocument

use of org.apache.solr.common.SolrInputDocument in project lucene-solr by apache.

the class PeerSyncTest method test.

@Test
@ShardsFixed(num = 3)
public void test() throws Exception {
    Set<Integer> docsAdded = new LinkedHashSet<>();
    handle.clear();
    handle.put("timestamp", SKIPVAL);
    handle.put("score", SKIPVAL);
    handle.put("maxScore", SKIPVAL);
    SolrClient client0 = clients.get(0);
    SolrClient client1 = clients.get(1);
    SolrClient client2 = clients.get(2);
    long v = 0;
    add(client0, seenLeader, sdoc("id", "1", "_version_", ++v));
    // this fails because client0 has no context (i.e. no updates of its own to judge if applying the updates
    // from client1 will bring it into sync with client1)
    assertSync(client1, numVersions, false, shardsArr[0]);
    // bring client1 back into sync with client0 by adding the doc
    add(client1, seenLeader, sdoc("id", "1", "_version_", v));
    // both have the same version list, so sync should now return true
    assertSync(client1, numVersions, true, shardsArr[0]);
    // TODO: test that updates weren't necessary
    client0.commit();
    client1.commit();
    queryAndCompare(params("q", "*:*"), client0, client1);
    add(client0, seenLeader, addRandFields(sdoc("id", "2", "_version_", ++v)));
    // now client1 has the context to sync
    assertSync(client1, numVersions, true, shardsArr[0]);
    client0.commit();
    client1.commit();
    queryAndCompare(params("q", "*:*"), client0, client1);
    add(client0, seenLeader, addRandFields(sdoc("id", "3", "_version_", ++v)));
    add(client0, seenLeader, addRandFields(sdoc("id", "4", "_version_", ++v)));
    add(client0, seenLeader, addRandFields(sdoc("id", "5", "_version_", ++v)));
    add(client0, seenLeader, addRandFields(sdoc("id", "6", "_version_", ++v)));
    add(client0, seenLeader, addRandFields(sdoc("id", "7", "_version_", ++v)));
    add(client0, seenLeader, addRandFields(sdoc("id", "8", "_version_", ++v)));
    add(client0, seenLeader, addRandFields(sdoc("id", "9", "_version_", ++v)));
    add(client0, seenLeader, addRandFields(sdoc("id", "10", "_version_", ++v)));
    for (int i = 0; i < 10; i++) docsAdded.add(i + 1);
    assertSync(client1, numVersions, true, shardsArr[0]);
    validateDocs(docsAdded, client0, client1);
    int toAdd = (int) (numVersions * .95);
    for (int i = 0; i < toAdd; i++) {
        add(client0, seenLeader, sdoc("id", Integer.toString(i + 11), "_version_", v + i + 1));
        docsAdded.add(i + 11);
    }
    // sync should fail since there's not enough overlap to give us confidence
    assertSync(client1, numVersions, false, shardsArr[0]);
    // add some of the docs that were missing... just enough to give enough overlap
    int toAdd2 = (int) (numVersions * .25);
    for (int i = 0; i < toAdd2; i++) {
        add(client1, seenLeader, sdoc("id", Integer.toString(i + 11), "_version_", v + i + 1));
    }
    assertSync(client1, numVersions, true, shardsArr[0]);
    validateDocs(docsAdded, client0, client1);
    // test delete and deleteByQuery
    v = 1000;
    SolrInputDocument doc = sdoc("id", "1000", "_version_", ++v);
    add(client0, seenLeader, doc);
    add(client0, seenLeader, sdoc("id", "1001", "_version_", ++v));
    delQ(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", Long.toString(-++v)), "id:1001 OR id:1002");
    add(client0, seenLeader, sdoc("id", "1002", "_version_", ++v));
    del(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", Long.toString(-++v)), "1000");
    // 1002 added
    docsAdded.add(1002);
    assertSync(client1, numVersions, true, shardsArr[0]);
    validateDocs(docsAdded, client0, client1);
    // test that delete by query is returned even if not requested, and that it doesn't delete newer stuff than it should
    v = 2000;
    SolrClient client = client0;
    add(client, seenLeader, sdoc("id", "2000", "_version_", ++v));
    add(client, seenLeader, sdoc("id", "2001", "_version_", ++v));
    delQ(client, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", Long.toString(-++v)), "id:2001 OR id:2002");
    add(client, seenLeader, sdoc("id", "2002", "_version_", ++v));
    del(client, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", Long.toString(-++v)), "2000");
    // 2002 added
    docsAdded.add(2002);
    v = 2000;
    client = client1;
    add(client, seenLeader, sdoc("id", "2000", "_version_", ++v));
    // pretend we missed the add of 2001.  peersync should retrieve it, but should also retrieve any deleteByQuery objects after it
    ++v;
    // add(client, seenLeader, sdoc("id","2001","_version_",++v));
    delQ(client, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", Long.toString(-++v)), "id:2001 OR id:2002");
    add(client, seenLeader, sdoc("id", "2002", "_version_", ++v));
    del(client, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", Long.toString(-++v)), "2000");
    assertSync(client1, numVersions, true, shardsArr[0]);
    validateDocs(docsAdded, client0, client1);
    //
    // Test that handling reorders work when applying docs retrieved from peer
    //
    // this should cause us to retrieve the delete (but not the following add)
    // the reorder in application shouldn't affect anything
    add(client0, seenLeader, sdoc("id", "3000", "_version_", 3001));
    add(client1, seenLeader, sdoc("id", "3000", "_version_", 3001));
    del(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "3000"), "3000");
    docsAdded.add(3000);
    // this should cause us to retrieve an add tha was previously deleted
    add(client0, seenLeader, sdoc("id", "3001", "_version_", 3003));
    del(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "3001"), "3004");
    del(client1, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "3001"), "3004");
    // this should cause us to retrieve an older add that was overwritten
    add(client0, seenLeader, sdoc("id", "3002", "_version_", 3004));
    add(client0, seenLeader, sdoc("id", "3002", "_version_", 3005));
    add(client1, seenLeader, sdoc("id", "3002", "_version_", 3005));
    // 3001 added
    docsAdded.add(3001);
    // 3002 added
    docsAdded.add(3002);
    assertSync(client1, numVersions, true, shardsArr[0]);
    validateDocs(docsAdded, client0, client1);
    // now lets check fingerprinting causes appropriate fails
    v = 4000;
    add(client0, seenLeader, sdoc("id", Integer.toString((int) v), "_version_", v));
    docsAdded.add(4000);
    toAdd = numVersions + 10;
    for (int i = 0; i < toAdd; i++) {
        add(client0, seenLeader, sdoc("id", Integer.toString((int) v + i + 1), "_version_", v + i + 1));
        add(client1, seenLeader, sdoc("id", Integer.toString((int) v + i + 1), "_version_", v + i + 1));
        docsAdded.add((int) v + i + 1);
    }
    // client0 now has an additional add beyond our window and the fingerprint should cause this to fail
    assertSync(client1, numVersions, false, shardsArr[0]);
    // if we turn of fingerprinting, it should succeed
    System.setProperty("solr.disableFingerprint", "true");
    try {
        assertSync(client1, numVersions, true, shardsArr[0]);
    } finally {
        System.clearProperty("solr.disableFingerprint");
    }
    // lets add the missing document and verify that order doesn't matter
    add(client1, seenLeader, sdoc("id", Integer.toString((int) v), "_version_", v));
    assertSync(client1, numVersions, true, shardsArr[0]);
    // lets do some overwrites to ensure that repeated updates and maxDoc don't matter
    for (int i = 0; i < 10; i++) {
        add(client0, seenLeader, sdoc("id", Integer.toString((int) v + i + 1), "_version_", v + i + 1));
    }
    assertSync(client1, numVersions, true, shardsArr[0]);
    validateDocs(docsAdded, client0, client1);
    // lets add some in-place updates
    // full update
    add(client0, seenLeader, sdoc("id", "5000", "val_i_dvo", 0, "title", "mytitle", "_version_", 5000));
    docsAdded.add(5000);
    assertSync(client1, numVersions, true, shardsArr[0]);
    // verify the in-place updated document (id=5000) has correct fields
    assertEquals(0, client1.getById("5000").get("val_i_dvo"));
    assertEquals(client0.getById("5000") + " and " + client1.getById("5000"), "mytitle", client1.getById("5000").getFirstValue("title"));
    ModifiableSolrParams inPlaceParams = new ModifiableSolrParams(seenLeader);
    inPlaceParams.set(DistributedUpdateProcessor.DISTRIB_INPLACE_PREVVERSION, "5000");
    // in-place update
    add(client0, inPlaceParams, sdoc("id", "5000", "val_i_dvo", 1, "_version_", 5001));
    assertSync(client1, numVersions, true, shardsArr[0]);
    // verify the in-place updated document (id=5000) has correct fields
    assertEquals(1, client1.getById("5000").get("val_i_dvo"));
    assertEquals(client0.getById("5000") + " and " + client1.getById("5000"), "mytitle", client1.getById("5000").getFirstValue("title"));
    // interleave the in-place updates with a few deletes to other documents
    del(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "5002"), 4001);
    delQ(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "5003"), "id:4002");
    docsAdded.remove(4001);
    docsAdded.remove(4002);
    inPlaceParams.set(DistributedUpdateProcessor.DISTRIB_INPLACE_PREVVERSION, "5001");
    // in-place update
    add(client0, inPlaceParams, sdoc("id", 5000, "val_i_dvo", 2, "_version_", 5004));
    assertSync(client1, numVersions, true, shardsArr[0]);
    // verify the in-place updated document (id=5000) has correct fields
    assertEquals(2, client1.getById("5000").get("val_i_dvo"));
    assertEquals(client0.getById("5000") + " and " + client1.getById("5000"), "mytitle", client1.getById("5000").getFirstValue("title"));
    // a DBQ with value
    // current val is 2, so this should not delete anything
    delQ(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "5005"), "val_i_dvo:1");
    assertSync(client1, numVersions, true, shardsArr[0]);
    // full update
    add(client0, seenLeader, sdoc("id", "5000", "val_i_dvo", 0, "title", "mytitle", "_version_", 5000));
    docsAdded.add(5000);
    assertSync(client1, numVersions, true, shardsArr[0]);
    inPlaceParams.set(DistributedUpdateProcessor.DISTRIB_INPLACE_PREVVERSION, "5004");
    add(client0, inPlaceParams, sdoc("id", 5000, "val_i_dvo", 3, "_version_", 5006));
    assertSync(client1, numVersions, true, shardsArr[0]);
    // verify the in-place updated document (id=5000) has correct fields
    assertEquals(3, client1.getById("5000").get("val_i_dvo"));
    assertEquals(client0.getById("5000") + " and " + client1.getById("5000"), "mytitle", client1.getById("5000").getFirstValue("title"));
    validateDocs(docsAdded, client0, client1);
    del(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "5007"), 5000);
    docsAdded.remove(5000);
    assertSync(client1, numVersions, true, shardsArr[0]);
    validateDocs(docsAdded, client0, client1);
    // if doc with id=6000 is deleted, further in-place-updates should fail
    // full update
    add(client0, seenLeader, sdoc("id", "6000", "val_i_dvo", 6, "title", "mytitle", "_version_", 6000));
    // current val is 6000, this will delete id=6000
    delQ(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "6004"), "val_i_dvo:6");
    assertSync(client1, numVersions, true, shardsArr[0]);
    SolrException ex = expectThrows(SolrException.class, () -> {
        inPlaceParams.set(DistributedUpdateProcessor.DISTRIB_INPLACE_PREVVERSION, "6000");
        add(client0, inPlaceParams, sdoc("id", 6000, "val_i_dvo", 6003, "_version_", 5007));
    });
    assertEquals(ex.toString(), SolrException.ErrorCode.SERVER_ERROR.code, ex.code());
    assertThat(ex.getMessage(), containsString("Can't find document with id=6000"));
    // Reordered DBQ with Child-nodes (SOLR-10114)
    docsAdded.clear();
    // Reordered full delete should not delete child-docs
    // add with later version
    add(client0, seenLeader, sdocWithChildren(7001, "7001", 2));
    docsAdded.add(7001);
    docsAdded.add(7001001);
    docsAdded.add(7001002);
    // reordered delete
    delQ(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "7000"), "id:*");
    assertSync(client1, numVersions, true, shardsArr[0]);
    validateDocs(docsAdded, client0, client1);
    // Reordered DBQ should not affect update
    // add with later version
    add(client0, seenLeader, sdocWithChildren(8000, "8000", 5));
    // not found, arrives earlier
    delQ(client0, params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "8002"), "id:8500");
    // update with two childs
    add(client0, seenLeader, sdocWithChildren(8000, "8001", 2));
    docsAdded.add(8000);
    docsAdded.add(8000001);
    docsAdded.add(8000002);
    assertSync(client1, numVersions, true, shardsArr[0]);
    validateDocs(docsAdded, client0, client1);
}

Example 48 with SolrInputDocument

use of org.apache.solr.common.SolrInputDocument in project lucene-solr by apache.

the class AddBlockUpdateTest method testJavaBinCodec.

@Test
public void testJavaBinCodec() throws IOException {
    //actually this test must be in other test class
    SolrInputDocument topDocument = new SolrInputDocument();
    topDocument.addField("parent_f1", "v1");
    topDocument.addField("parent_f2", "v2");
    int childsNum = atLeast(10);
    for (int index = 0; index < childsNum; ++index) {
        addChildren("child", topDocument, index, false);
    }
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    new JavaBinCodec().marshal(topDocument, os);
    byte[] buffer = os.toByteArray();
    //now read the Object back
    InputStream is = new ByteArrayInputStream(buffer);
    SolrInputDocument result = (SolrInputDocument) new JavaBinCodec().unmarshal(is);
    assertEquals(2, result.size());
    assertEquals("v1", result.getFieldValue("parent_f1"));
    assertEquals("v2", result.getFieldValue("parent_f2"));
    List<SolrInputDocument> resultChilds = result.getChildDocuments();
    int resultChildsSize = resultChilds == null ? 0 : resultChilds.size();
    assertEquals(childsNum, resultChildsSize);
    for (int childIndex = 0; childIndex < childsNum; ++childIndex) {
        SolrInputDocument child = resultChilds.get(childIndex);
        for (int fieldNum = 0; fieldNum < childIndex; ++fieldNum) {
            assertEquals(childIndex + "value" + fieldNum, child.getFieldValue(childIndex + "child" + fieldNum));
        }
        List<SolrInputDocument> grandChilds = child.getChildDocuments();
        int grandChildsSize = grandChilds == null ? 0 : grandChilds.size();
        assertEquals(childIndex * 2, grandChildsSize);
        for (int grandIndex = 0; grandIndex < childIndex * 2; ++grandIndex) {
            SolrInputDocument grandChild = grandChilds.get(grandIndex);
            assertFalse(grandChild.hasChildDocuments());
            for (int fieldNum = 0; fieldNum < grandIndex; ++fieldNum) {
                assertEquals(grandIndex + "value" + fieldNum, grandChild.getFieldValue(grandIndex + "grand" + fieldNum));
            }
        }
    }
}

Example 49 with SolrInputDocument

use of org.apache.solr.common.SolrInputDocument in project lucene-solr by apache.

the class AddBlockUpdateTest method testSolrJXML.

@SuppressWarnings("serial")
@Test
public void testSolrJXML() throws IOException {
    UpdateRequest req = new UpdateRequest();
    List<SolrInputDocument> docs = new ArrayList<>();
    SolrInputDocument document1 = new SolrInputDocument() {

        {
            final String id = id();
            addField("id", id);
            addField("parent_s", "X");
            ArrayList<SolrInputDocument> ch1 = new ArrayList<>(Arrays.asList(new SolrInputDocument() {

                {
                    addField("id", id());
                    addField("child_s", "y");
                }
            }, new SolrInputDocument() {

                {
                    addField("id", id());
                    addField("child_s", "z");
                }
            }));
            Collections.shuffle(ch1, random());
            addChildDocuments(ch1);
        }
    };
    SolrInputDocument document2 = new SolrInputDocument() {

        {
            final String id = id();
            addField("id", id);
            addField("parent_s", "A");
            addChildDocument(new SolrInputDocument() {

                {
                    addField("id", id());
                    addField("child_s", "b");
                }
            });
            addChildDocument(new SolrInputDocument() {

                {
                    addField("id", id());
                    addField("child_s", "c");
                }
            });
        }
    };
    docs.add(document1);
    docs.add(document2);
    Collections.shuffle(docs, random());
    req.add(docs);
    RequestWriter requestWriter = new RequestWriter();
    OutputStream os = new ByteArrayOutputStream();
    requestWriter.write(req, os);
    assertBlockU(os.toString());
    assertU(commit());
    final SolrIndexSearcher searcher = getSearcher();
    assertSingleParentOf(searcher, one("yz"), "X");
    assertSingleParentOf(searcher, one("bc"), "A");
}

Example 50 with SolrInputDocument

use of org.apache.solr.common.SolrInputDocument in project lucene-solr by apache.

the class CdcrUpdateLogTest method testSubReader.

@Test
public void testSubReader() throws Exception {
    this.clearCore();
    CdcrUpdateLog ulog = (CdcrUpdateLog) h.getCore().getUpdateHandler().getUpdateLog();
    File logDir = new File(h.getCore().getUpdateHandler().getUpdateLog().getLogDir());
    CdcrUpdateLog.CdcrLogReader reader = ulog.newLogReader();
    int start = 0;
    LinkedList<Long> versions = new LinkedList<>();
    addDocs(10, start, versions);
    start += 10;
    assertU(commit());
    addDocs(10, start, versions);
    start += 10;
    assertU(commit());
    assertEquals(2, ulog.getLogList(logDir).length);
    // start to read the first tlog
    for (int i = 0; i < 10; i++) {
        assertNotNull(reader.next());
    }
    // instantiate a sub reader, and finish to read the first tlog (commit operation), plus start to read the
    // second tlog (first five adds)
    CdcrUpdateLog.CdcrLogReader subReader = reader.getSubReader();
    for (int i = 0; i < 6; i++) {
        assertNotNull(subReader.next());
    }
    // Five adds + one commit
    assertEquals(6, subReader.getNumberOfRemainingRecords());
    // Generate a new tlog
    addDocs(105, start, versions);
    start += 105;
    assertU(commit());
    // Even if the subreader is past the first tlog, the first tlog should not have been removed
    // since the parent reader is still pointing to it
    assertEquals(3, ulog.getLogList(logDir).length);
    // fast forward the parent reader with the subreader
    reader.forwardSeek(subReader);
    subReader.close();
    // After fast forward, the parent reader should be position on the doc15
    List o = (List) reader.next();
    assertNotNull(o);
    assertTrue(o.get(2) instanceof SolrInputDocument);
    assertEquals("15", ((SolrInputDocument) o.get(2)).getFieldValue("id"));
    // Finish to read the second tlog, and start to read the third one
    for (int i = 0; i < 6; i++) {
        assertNotNull(reader.next());
    }
    assertEquals(105, reader.getNumberOfRemainingRecords());
    // Generate a new tlog to activate tlog cleaning
    addDocs(10, start, versions);
    start += 10;
    assertU(commit());
    // If the parent reader was correctly fast forwarded, it should be on the third tlog, and the first two should
    // have been removed.
    assertEquals(2, ulog.getLogList(logDir).length);
}