Examples with SortedNumericDocValuesField org.apache.lucene.document.SortedNumericDocValuesField used on opensource projects

Example 46 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class TestSortedNumericSortField method testMissingLast.

public void testMissingLast() throws Exception {
    Directory dir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    Document doc = new Document();
    doc.add(new SortedNumericDocValuesField("value", 5));
    doc.add(newStringField("id", "2", Field.Store.YES));
    writer.addDocument(doc);
    doc = new Document();
    doc.add(new SortedNumericDocValuesField("value", 3));
    doc.add(new SortedNumericDocValuesField("value", 7));
    doc.add(newStringField("id", "1", Field.Store.YES));
    writer.addDocument(doc);
    doc = new Document();
    doc.add(newStringField("id", "3", Field.Store.YES));
    writer.addDocument(doc);
    IndexReader ir = writer.getReader();
    writer.close();
    IndexSearcher searcher = newSearcher(ir);
    SortField sortField = new SortedNumericSortField("value", SortField.Type.INT);
    sortField.setMissingValue(Integer.MAX_VALUE);
    Sort sort = new Sort(sortField);
    TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
    assertEquals(3, td.totalHits);
    // 3 comes before 5
    assertEquals("1", searcher.doc(td.scoreDocs[0].doc).get("id"));
    assertEquals("2", searcher.doc(td.scoreDocs[1].doc).get("id"));
    // null comes last
    assertEquals("3", searcher.doc(td.scoreDocs[2].doc).get("id"));
    ir.close();
    dir.close();
}

Example 47 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class TestJoinUtil method testEquals_numericJoin.

public void testEquals_numericJoin() throws Exception {
    final int numDocs = atLeast(random(), 50);
    try (final Directory dir = newDirectory()) {
        try (final RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()))) {
            boolean multiValued = random().nextBoolean();
            String joinField = multiValued ? "mvField" : "svField";
            for (int id = 0; id < numDocs; id++) {
                Document doc = new Document();
                doc.add(new TextField("id", "" + id, Field.Store.NO));
                doc.add(new TextField("name", "name" + (id % 7), Field.Store.NO));
                if (multiValued) {
                    int numValues = 1 + random().nextInt(2);
                    for (int i = 0; i < numValues; i++) {
                        doc.add(new IntPoint(joinField, random().nextInt(13)));
                        doc.add(new SortedNumericDocValuesField(joinField, random().nextInt(13)));
                    }
                } else {
                    doc.add(new IntPoint(joinField, random().nextInt(13)));
                    doc.add(new NumericDocValuesField(joinField, random().nextInt(13)));
                }
                w.addDocument(doc);
            }
            Set<ScoreMode> scoreModes = EnumSet.allOf(ScoreMode.class);
            ScoreMode scoreMode1 = scoreModes.toArray(new ScoreMode[0])[random().nextInt(scoreModes.size())];
            scoreModes.remove(scoreMode1);
            ScoreMode scoreMode2 = scoreModes.toArray(new ScoreMode[0])[random().nextInt(scoreModes.size())];
            final Query x;
            try (IndexReader r = w.getReader()) {
                IndexSearcher indexSearcher = new IndexSearcher(r);
                x = JoinUtil.createJoinQuery(joinField, multiValued, joinField, Integer.class, new TermQuery(new Term("name", "name5")), indexSearcher, scoreMode1);
                assertEquals("identical calls to createJoinQuery", x, JoinUtil.createJoinQuery(joinField, multiValued, joinField, Integer.class, new TermQuery(new Term("name", "name5")), indexSearcher, scoreMode1));
                assertFalse("score mode (" + scoreMode1 + " != " + scoreMode2 + "), but queries are equal", x.equals(JoinUtil.createJoinQuery(joinField, multiValued, joinField, Integer.class, new TermQuery(new Term("name", "name5")), indexSearcher, scoreMode2)));
                assertFalse("from fields (joinField != \"other_field\") but queries equals", x.equals(JoinUtil.createJoinQuery(joinField, multiValued, "other_field", Integer.class, new TermQuery(new Term("name", "name5")), indexSearcher, scoreMode1)));
                assertFalse("from fields (\"other_field\" != joinField) but queries equals", x.equals(JoinUtil.createJoinQuery("other_field", multiValued, joinField, Integer.class, new TermQuery(new Term("name", "name5")), indexSearcher, scoreMode1)));
                assertFalse("fromQuery (name:name5 != name:name6) but queries equals", x.equals(JoinUtil.createJoinQuery("other_field", multiValued, joinField, Integer.class, new TermQuery(new Term("name", "name6")), indexSearcher, scoreMode1)));
            }
            for (int i = 14; i < 26; i++) {
                Document doc = new Document();
                doc.add(new TextField("id", "new_id", Field.Store.NO));
                doc.add(new TextField("name", "name5", Field.Store.NO));
                if (multiValued) {
                    int numValues = 1 + random().nextInt(2);
                    for (int j = 0; j < numValues; j++) {
                        doc.add(new SortedNumericDocValuesField(joinField, i));
                        doc.add(new IntPoint(joinField, i));
                    }
                } else {
                    doc.add(new NumericDocValuesField(joinField, i));
                    doc.add(new IntPoint(joinField, i));
                }
                w.addDocument(doc);
            }
            try (IndexReader r = w.getReader()) {
                IndexSearcher indexSearcher = new IndexSearcher(r);
                assertFalse("Query shouldn't be equal, because new join values have been indexed", x.equals(JoinUtil.createJoinQuery(joinField, multiValued, joinField, Integer.class, new TermQuery(new Term("name", "name5")), indexSearcher, scoreMode1)));
            }
        }
    }
}

Example 48 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class TestMemoryIndexAgainstRAMDir method testDocValuesMemoryIndexVsNormalIndex.

public void testDocValuesMemoryIndexVsNormalIndex() throws Exception {
    Document doc = new Document();
    long randomLong = random().nextLong();
    doc.add(new NumericDocValuesField("numeric", randomLong));
    int numValues = atLeast(5);
    for (int i = 0; i < numValues; i++) {
        randomLong = random().nextLong();
        doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong));
        if (random().nextBoolean()) {
            // randomly duplicate field/value
            doc.add(new SortedNumericDocValuesField("sorted_numeric", randomLong));
        }
    }
    BytesRef randomTerm = new BytesRef(randomTerm());
    doc.add(new BinaryDocValuesField("binary", randomTerm));
    if (random().nextBoolean()) {
        doc.add(new StringField("binary", randomTerm, Field.Store.NO));
    }
    randomTerm = new BytesRef(randomTerm());
    doc.add(new SortedDocValuesField("sorted", randomTerm));
    if (random().nextBoolean()) {
        doc.add(new StringField("sorted", randomTerm, Field.Store.NO));
    }
    numValues = atLeast(5);
    for (int i = 0; i < numValues; i++) {
        randomTerm = new BytesRef(randomTerm());
        doc.add(new SortedSetDocValuesField("sorted_set", randomTerm));
        if (random().nextBoolean()) {
            // randomly duplicate field/value
            doc.add(new SortedSetDocValuesField("sorted_set", randomTerm));
        }
        if (random().nextBoolean()) {
            // randomily just add a normal string field
            doc.add(new StringField("sorted_set", randomTerm, Field.Store.NO));
        }
    }
    MockAnalyzer mockAnalyzer = new MockAnalyzer(random());
    MemoryIndex memoryIndex = MemoryIndex.fromDocument(doc, mockAnalyzer);
    IndexReader indexReader = memoryIndex.createSearcher().getIndexReader();
    LeafReader leafReader = indexReader.leaves().get(0).reader();
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer));
    writer.addDocument(doc);
    writer.close();
    IndexReader controlIndexReader = DirectoryReader.open(dir);
    LeafReader controlLeafReader = controlIndexReader.leaves().get(0).reader();
    NumericDocValues numericDocValues = leafReader.getNumericDocValues("numeric");
    NumericDocValues controlNumericDocValues = controlLeafReader.getNumericDocValues("numeric");
    assertEquals(0, numericDocValues.nextDoc());
    assertEquals(0, controlNumericDocValues.nextDoc());
    assertEquals(controlNumericDocValues.longValue(), numericDocValues.longValue());
    SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric");
    assertEquals(0, sortedNumericDocValues.nextDoc());
    SortedNumericDocValues controlSortedNumericDocValues = controlLeafReader.getSortedNumericDocValues("sorted_numeric");
    assertEquals(0, controlSortedNumericDocValues.nextDoc());
    assertEquals(controlSortedNumericDocValues.docValueCount(), sortedNumericDocValues.docValueCount());
    for (int i = 0; i < controlSortedNumericDocValues.docValueCount(); i++) {
        assertEquals(controlSortedNumericDocValues.nextValue(), sortedNumericDocValues.nextValue());
    }
    BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("binary");
    BinaryDocValues controlBinaryDocValues = controlLeafReader.getBinaryDocValues("binary");
    assertEquals(0, binaryDocValues.nextDoc());
    assertEquals(0, controlBinaryDocValues.nextDoc());
    assertEquals(controlBinaryDocValues.binaryValue(), binaryDocValues.binaryValue());
    SortedDocValues sortedDocValues = leafReader.getSortedDocValues("sorted");
    SortedDocValues controlSortedDocValues = controlLeafReader.getSortedDocValues("sorted");
    assertEquals(controlSortedDocValues.getValueCount(), sortedDocValues.getValueCount());
    assertEquals(0, sortedDocValues.nextDoc());
    assertEquals(0, controlSortedDocValues.nextDoc());
    assertEquals(controlSortedDocValues.binaryValue(), sortedDocValues.binaryValue());
    assertEquals(controlSortedDocValues.ordValue(), sortedDocValues.ordValue());
    assertEquals(controlSortedDocValues.lookupOrd(0), sortedDocValues.lookupOrd(0));
    SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
    assertEquals(0, sortedSetDocValues.nextDoc());
    SortedSetDocValues controlSortedSetDocValues = controlLeafReader.getSortedSetDocValues("sorted_set");
    assertEquals(0, controlSortedSetDocValues.nextDoc());
    assertEquals(controlSortedSetDocValues.getValueCount(), sortedSetDocValues.getValueCount());
    for (long controlOrd = controlSortedSetDocValues.nextOrd(); controlOrd != SortedSetDocValues.NO_MORE_ORDS; controlOrd = controlSortedSetDocValues.nextOrd()) {
        assertEquals(controlOrd, sortedSetDocValues.nextOrd());
        assertEquals(controlSortedSetDocValues.lookupOrd(controlOrd), sortedSetDocValues.lookupOrd(controlOrd));
    }
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd());
    indexReader.close();
    controlIndexReader.close();
    dir.close();
}

Example 49 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class PointField method createFields.

@Override
public List<IndexableField> createFields(SchemaField sf, Object value) {
    if (!isFieldUsed(sf)) {
        return Collections.emptyList();
    }
    List<IndexableField> fields = new ArrayList<>(3);
    IndexableField field = null;
    if (sf.indexed()) {
        field = createField(sf, value);
        fields.add(field);
    }
    if (sf.hasDocValues()) {
        final Number numericValue;
        if (field == null) {
            final Object nativeTypeObject = toNativeType(value);
            if (getNumberType() == NumberType.DATE) {
                numericValue = ((Date) nativeTypeObject).getTime();
            } else {
                numericValue = (Number) nativeTypeObject;
            }
        } else {
            numericValue = field.numericValue();
        }
        final long bits;
        if (!sf.multiValued()) {
            if (numericValue instanceof Integer || numericValue instanceof Long) {
                bits = numericValue.longValue();
            } else if (numericValue instanceof Float) {
                bits = Float.floatToIntBits(numericValue.floatValue());
            } else {
                assert numericValue instanceof Double;
                bits = Double.doubleToLongBits(numericValue.doubleValue());
            }
            fields.add(new NumericDocValuesField(sf.getName(), bits));
        } else {
            // MultiValued
            if (numericValue instanceof Integer || numericValue instanceof Long) {
                bits = numericValue.longValue();
            } else if (numericValue instanceof Float) {
                bits = NumericUtils.floatToSortableInt(numericValue.floatValue());
            } else {
                assert numericValue instanceof Double;
                bits = NumericUtils.doubleToSortableLong(numericValue.doubleValue());
            }
            fields.add(new SortedNumericDocValuesField(sf.getName(), bits));
        }
    }
    if (sf.stored()) {
        fields.add(getStoredField(sf, value));
    }
    return fields;
}

Example 50 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project crate by crate.

the class ShortColumnReferenceTest method insertValues.

@Override
protected void insertValues(IndexWriter writer) throws Exception {
    for (short i = -10; i < 10; i++) {
        Document doc = new Document();
        doc.add(new StringField("_id", Short.toString(i), Field.Store.NO));
        doc.add(new SortedNumericDocValuesField(column, i));
        writer.addDocument(doc);
    }
}