Examples with SortedDocValuesField org.apache.lucene.document.SortedDocValuesField used on opensource projects

Example 61 with SortedDocValuesField

use of org.apache.lucene.document.SortedDocValuesField in project lucene-solr by apache.

the class TestBinaryDocValuesUpdates method testMultipleDocValuesTypes.

public void testMultipleDocValuesTypes() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    // prevent merges
    conf.setMaxBufferedDocs(10);
    IndexWriter writer = new IndexWriter(dir, conf);
    for (int i = 0; i < 4; i++) {
        Document doc = new Document();
        doc.add(new StringField("dvUpdateKey", "dv", Store.NO));
        doc.add(new NumericDocValuesField("ndv", i));
        doc.add(new BinaryDocValuesField("bdv", new BytesRef(Integer.toString(i))));
        doc.add(new SortedDocValuesField("sdv", new BytesRef(Integer.toString(i))));
        doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(Integer.toString(i))));
        doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(Integer.toString(i * 2))));
        writer.addDocument(doc);
    }
    writer.commit();
    // update all docs' bdv field
    writer.updateBinaryDocValue(new Term("dvUpdateKey", "dv"), "bdv", toBytes(17L));
    writer.close();
    final DirectoryReader reader = DirectoryReader.open(dir);
    LeafReader r = reader.leaves().get(0).reader();
    NumericDocValues ndv = r.getNumericDocValues("ndv");
    BinaryDocValues bdv = r.getBinaryDocValues("bdv");
    SortedDocValues sdv = r.getSortedDocValues("sdv");
    SortedSetDocValues ssdv = r.getSortedSetDocValues("ssdv");
    for (int i = 0; i < r.maxDoc(); i++) {
        assertEquals(i, ndv.nextDoc());
        assertEquals(i, ndv.longValue());
        assertEquals(i, bdv.nextDoc());
        assertEquals(17, getValue(bdv));
        assertEquals(i, sdv.nextDoc());
        BytesRef term = sdv.binaryValue();
        assertEquals(new BytesRef(Integer.toString(i)), term);
        assertEquals(i, ssdv.nextDoc());
        long ord = ssdv.nextOrd();
        term = ssdv.lookupOrd(ord);
        assertEquals(i, Integer.parseInt(term.utf8ToString()));
        // For the i=0 case, we added the same value twice, which was dedup'd by IndexWriter so it has only one value:
        if (i != 0) {
            ord = ssdv.nextOrd();
            term = ssdv.lookupOrd(ord);
            assertEquals(i * 2, Integer.parseInt(term.utf8ToString()));
        }
        assertEquals(SortedSetDocValues.NO_MORE_ORDS, ssdv.nextOrd());
    }
    reader.close();
    dir.close();
}

Example 62 with SortedDocValuesField

use of org.apache.lucene.document.SortedDocValuesField in project lucene-solr by apache.

the class TestNumericDocValuesUpdates method testMultipleDocValuesTypes.

@Test
public void testMultipleDocValuesTypes() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    // prevent merges
    conf.setMaxBufferedDocs(10);
    IndexWriter writer = new IndexWriter(dir, conf);
    for (int i = 0; i < 4; i++) {
        Document doc = new Document();
        doc.add(new StringField("dvUpdateKey", "dv", Store.NO));
        doc.add(new NumericDocValuesField("ndv", i));
        doc.add(new BinaryDocValuesField("bdv", new BytesRef(Integer.toString(i))));
        doc.add(new SortedDocValuesField("sdv", new BytesRef(Integer.toString(i))));
        doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(Integer.toString(i))));
        doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(Integer.toString(i * 2))));
        writer.addDocument(doc);
    }
    writer.commit();
    // update all docs' ndv field
    writer.updateNumericDocValue(new Term("dvUpdateKey", "dv"), "ndv", 17L);
    writer.close();
    final DirectoryReader reader = DirectoryReader.open(dir);
    LeafReader r = reader.leaves().get(0).reader();
    NumericDocValues ndv = r.getNumericDocValues("ndv");
    BinaryDocValues bdv = r.getBinaryDocValues("bdv");
    SortedDocValues sdv = r.getSortedDocValues("sdv");
    SortedSetDocValues ssdv = r.getSortedSetDocValues("ssdv");
    for (int i = 0; i < r.maxDoc(); i++) {
        assertEquals(i, ndv.nextDoc());
        assertEquals(17, ndv.longValue());
        assertEquals(i, bdv.nextDoc());
        BytesRef term = bdv.binaryValue();
        assertEquals(new BytesRef(Integer.toString(i)), term);
        assertEquals(i, sdv.nextDoc());
        term = sdv.binaryValue();
        assertEquals(new BytesRef(Integer.toString(i)), term);
        assertEquals(i, ssdv.nextDoc());
        long ord = ssdv.nextOrd();
        term = ssdv.lookupOrd(ord);
        assertEquals(i, Integer.parseInt(term.utf8ToString()));
        if (i != 0) {
            ord = ssdv.nextOrd();
            term = ssdv.lookupOrd(ord);
            assertEquals(i * 2, Integer.parseInt(term.utf8ToString()));
        }
        assertEquals(SortedSetDocValues.NO_MORE_ORDS, ssdv.nextOrd());
    }
    reader.close();
    dir.close();
}

Example 63 with SortedDocValuesField

use of org.apache.lucene.document.SortedDocValuesField in project lucene-solr by apache.

the class TestSearchAfter method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    allSortFields = new ArrayList<>(Arrays.asList(new SortField[] { new SortField("int", SortField.Type.INT, false), new SortField("long", SortField.Type.LONG, false), new SortField("float", SortField.Type.FLOAT, false), new SortField("double", SortField.Type.DOUBLE, false), new SortField("bytes", SortField.Type.STRING, false), new SortField("bytesval", SortField.Type.STRING_VAL, false), new SortField("int", SortField.Type.INT, true), new SortField("long", SortField.Type.LONG, true), new SortField("float", SortField.Type.FLOAT, true), new SortField("double", SortField.Type.DOUBLE, true), new SortField("bytes", SortField.Type.STRING, true), new SortField("bytesval", SortField.Type.STRING_VAL, true), SortField.FIELD_SCORE, SortField.FIELD_DOC }));
    // Also test missing first / last for the "string" sorts:
    for (String field : new String[] { "bytes", "sortedbytesdocvalues" }) {
        for (int rev = 0; rev < 2; rev++) {
            boolean reversed = rev == 0;
            SortField sf = new SortField(field, SortField.Type.STRING, reversed);
            sf.setMissingValue(SortField.STRING_FIRST);
            allSortFields.add(sf);
            sf = new SortField(field, SortField.Type.STRING, reversed);
            sf.setMissingValue(SortField.STRING_LAST);
            allSortFields.add(sf);
        }
    }
    // Also test missing first / last for the "string_val" sorts:
    for (String field : new String[] { "sortedbytesdocvaluesval", "straightbytesdocvalues" }) {
        for (int rev = 0; rev < 2; rev++) {
            boolean reversed = rev == 0;
            SortField sf = new SortField(field, SortField.Type.STRING_VAL, reversed);
            sf.setMissingValue(SortField.STRING_FIRST);
            allSortFields.add(sf);
            sf = new SortField(field, SortField.Type.STRING_VAL, reversed);
            sf.setMissingValue(SortField.STRING_LAST);
            allSortFields.add(sf);
        }
    }
    int limit = allSortFields.size();
    for (int i = 0; i < limit; i++) {
        SortField sf = allSortFields.get(i);
        if (sf.getType() == SortField.Type.INT) {
            SortField sf2 = new SortField(sf.getField(), SortField.Type.INT, sf.getReverse());
            sf2.setMissingValue(random().nextInt());
            allSortFields.add(sf2);
        } else if (sf.getType() == SortField.Type.LONG) {
            SortField sf2 = new SortField(sf.getField(), SortField.Type.LONG, sf.getReverse());
            sf2.setMissingValue(random().nextLong());
            allSortFields.add(sf2);
        } else if (sf.getType() == SortField.Type.FLOAT) {
            SortField sf2 = new SortField(sf.getField(), SortField.Type.FLOAT, sf.getReverse());
            sf2.setMissingValue(random().nextFloat());
            allSortFields.add(sf2);
        } else if (sf.getType() == SortField.Type.DOUBLE) {
            SortField sf2 = new SortField(sf.getField(), SortField.Type.DOUBLE, sf.getReverse());
            sf2.setMissingValue(random().nextDouble());
            allSortFields.add(sf2);
        }
    }
    dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    int numDocs = atLeast(200);
    Random r = random();
    for (int i = 0; i < numDocs; i++) {
        List<Field> fields = new ArrayList<>();
        fields.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
        fields.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
        fields.add(new NumericDocValuesField("byte", (byte) r.nextInt()));
        fields.add(new NumericDocValuesField("short", (short) r.nextInt()));
        fields.add(new NumericDocValuesField("int", r.nextInt()));
        fields.add(new NumericDocValuesField("long", r.nextLong()));
        fields.add(new FloatDocValuesField("float", r.nextFloat()));
        fields.add(new DoubleDocValuesField("double", r.nextDouble()));
        fields.add(new SortedDocValuesField("bytes", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        fields.add(new BinaryDocValuesField("bytesval", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        Document document = new Document();
        document.add(new StoredField("id", "" + i));
        if (VERBOSE) {
            System.out.println("  add doc id=" + i);
        }
        for (Field field : fields) {
            // So we are sometimes missing that field:
            if (random().nextInt(5) != 4) {
                document.add(field);
                if (VERBOSE) {
                    System.out.println("    " + field);
                }
            }
        }
        iw.addDocument(document);
        if (random().nextInt(50) == 17) {
            iw.commit();
        }
    }
    reader = iw.getReader();
    iw.close();
    searcher = newSearcher(reader);
    if (VERBOSE) {
        System.out.println("  searcher=" + searcher);
    }
}

Example 64 with SortedDocValuesField

use of org.apache.lucene.document.SortedDocValuesField in project lucene-solr by apache.

the class TestRegexpRandom2 method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    dir = newDirectory();
    // sometimes use an empty string as field name
    fieldName = random().nextBoolean() ? "field" : "";
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)).setMaxBufferedDocs(TestUtil.nextInt(random(), 50, 1000)));
    Document doc = new Document();
    Field field = newStringField(fieldName, "", Field.Store.NO);
    doc.add(field);
    Field dvField = new SortedDocValuesField(fieldName, new BytesRef());
    doc.add(dvField);
    List<String> terms = new ArrayList<>();
    int num = atLeast(200);
    for (int i = 0; i < num; i++) {
        String s = TestUtil.randomUnicodeString(random());
        field.setStringValue(s);
        dvField.setBytesValue(new BytesRef(s));
        terms.add(s);
        writer.addDocument(doc);
    }
    if (VERBOSE) {
        // utf16 order
        Collections.sort(terms);
        System.out.println("UTF16 order:");
        for (String s : terms) {
            System.out.println("  " + UnicodeUtil.toHexString(s));
        }
    }
    reader = writer.getReader();
    searcher1 = newSearcher(reader);
    searcher2 = newSearcher(reader);
    writer.close();
}

Example 65 with SortedDocValuesField

use of org.apache.lucene.document.SortedDocValuesField in project lucene-solr by apache.

the class TestDocValues method testSortedField.

/** 
   * field with sorted docvalues
   */
public void testSortedField() throws Exception {
    Directory dir = newDirectory();
    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
    Document doc = new Document();
    doc.add(new SortedDocValuesField("foo", new BytesRef("bar")));
    iw.addDocument(doc);
    DirectoryReader dr = DirectoryReader.open(iw);
    LeafReader r = getOnlyLeafReader(dr);
    // ok
    assertNotNull(DocValues.getBinary(r, "foo"));
    assertNotNull(DocValues.getSorted(r, "foo"));
    assertNotNull(DocValues.getSortedSet(r, "foo"));
    // errors
    expectThrows(IllegalStateException.class, () -> {
        DocValues.getNumeric(r, "foo");
    });
    expectThrows(IllegalStateException.class, () -> {
        DocValues.getSortedNumeric(r, "foo");
    });
    dr.close();
    iw.close();
    dir.close();
}