Examples with SortedNumericSortField org.apache.lucene.search.SortedNumericSortField used on opensource projects

Example 11 with SortedNumericSortField

use of org.apache.lucene.search.SortedNumericSortField in project lucene-solr by apache.

the class TestIndexSorting method testMissingMultiValuedLongLast.

public void testMissingMultiValuedLongLast() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
    sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
    Sort indexSort = new Sort(sortField);
    iwc.setIndexSort(indexSort);
    IndexWriter w = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new NumericDocValuesField("id", 2));
    doc.add(new SortedNumericDocValuesField("foo", 18));
    doc.add(new SortedNumericDocValuesField("foo", 65));
    w.addDocument(doc);
    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
    w.commit();
    // missing
    doc = new Document();
    doc.add(new NumericDocValuesField("id", 3));
    w.addDocument(doc);
    w.commit();
    doc = new Document();
    doc.add(new NumericDocValuesField("id", 1));
    doc.add(new SortedNumericDocValuesField("foo", 7));
    doc.add(new SortedNumericDocValuesField("foo", 34));
    doc.add(new SortedNumericDocValuesField("foo", 74));
    w.addDocument(doc);
    w.forceMerge(1);
    DirectoryReader r = DirectoryReader.open(w);
    LeafReader leaf = getOnlyLeafReader(r);
    assertEquals(3, leaf.maxDoc());
    NumericDocValues values = leaf.getNumericDocValues("id");
    assertEquals(0, values.nextDoc());
    assertEquals(1, values.longValue());
    assertEquals(1, values.nextDoc());
    assertEquals(2, values.longValue());
    assertEquals(2, values.nextDoc());
    assertEquals(3, values.longValue());
    r.close();
    w.close();
    dir.close();
}

Example 12 with SortedNumericSortField

use of org.apache.lucene.search.SortedNumericSortField in project lucene-solr by apache.

the class TestIndexSorting method testBasicMultiValuedFloat.

public void testBasicMultiValuedFloat() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.FLOAT));
    iwc.setIndexSort(indexSort);
    IndexWriter w = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new NumericDocValuesField("id", 3));
    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(29.0f)));
    w.addDocument(doc);
    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
    w.commit();
    doc = new Document();
    doc.add(new NumericDocValuesField("id", 1));
    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(-1.0f)));
    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(34.0f)));
    w.addDocument(doc);
    w.commit();
    doc = new Document();
    doc.add(new NumericDocValuesField("id", 2));
    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
    w.addDocument(doc);
    w.forceMerge(1);
    DirectoryReader r = DirectoryReader.open(w);
    LeafReader leaf = getOnlyLeafReader(r);
    assertEquals(3, leaf.maxDoc());
    NumericDocValues values = leaf.getNumericDocValues("id");
    assertEquals(0, values.nextDoc());
    assertEquals(1, values.longValue());
    assertEquals(1, values.nextDoc());
    assertEquals(2, values.longValue());
    assertEquals(2, values.nextDoc());
    assertEquals(3, values.longValue());
    r.close();
    w.close();
    dir.close();
}

Example 13 with SortedNumericSortField

use of org.apache.lucene.search.SortedNumericSortField in project lucene-solr by apache.

the class TestIndexSorting method testMultiValuedRandom1.

public void testMultiValuedRandom1() throws IOException {
    boolean withDeletes = random().nextBoolean();
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.LONG));
    iwc.setIndexSort(indexSort);
    IndexWriter w = new IndexWriter(dir, iwc);
    final int numDocs = atLeast(1000);
    final FixedBitSet deleted = new FixedBitSet(numDocs);
    for (int i = 0; i < numDocs; ++i) {
        Document doc = new Document();
        int num = random().nextInt(10);
        for (int j = 0; j < num; j++) {
            doc.add(new SortedNumericDocValuesField("foo", random().nextInt(2000)));
        }
        doc.add(new StringField("id", Integer.toString(i), Store.YES));
        doc.add(new NumericDocValuesField("id", i));
        w.addDocument(doc);
        if (random().nextInt(5) == 0) {
            w.getReader().close();
        } else if (random().nextInt(30) == 0) {
            w.forceMerge(2);
        } else if (random().nextInt(4) == 0) {
            final int id = TestUtil.nextInt(random(), 0, i);
            deleted.set(id);
            w.deleteDocuments(new Term("id", Integer.toString(id)));
        }
    }
    DirectoryReader reader = w.getReader();
    // Now check that the index is consistent
    IndexSearcher searcher = newSearcher(reader);
    for (int i = 0; i < numDocs; ++i) {
        TermQuery termQuery = new TermQuery(new Term("id", Integer.toString(i)));
        final TopDocs topDocs = searcher.search(termQuery, 1);
        if (deleted.get(i)) {
            assertEquals(0, topDocs.totalHits);
        } else {
            assertEquals(1, topDocs.totalHits);
            NumericDocValues values = MultiDocValues.getNumericValues(reader, "id");
            assertEquals(topDocs.scoreDocs[0].doc, values.advance(topDocs.scoreDocs[0].doc));
            assertEquals(i, values.longValue());
            Document document = reader.document(topDocs.scoreDocs[0].doc);
            assertEquals(Integer.toString(i), document.get("id"));
        }
    }
    reader.close();
    w.close();
    dir.close();
}

Example 14 with SortedNumericSortField

use of org.apache.lucene.search.SortedNumericSortField in project lucene-solr by apache.

the class SortedNumericDocValuesWriter method getDocComparator.

@Override
Sorter.DocComparator getDocComparator(int maxDoc, SortField sortField) throws IOException {
    assert sortField instanceof SortedNumericSortField;
    assert finalValues == null && finalValuesCount == null;
    finalValues = pending.build();
    finalValuesCount = pendingCounts.build();
    final SortedNumericDocValues docValues = new BufferedSortedNumericDocValues(finalValues, finalValuesCount, docsWithField.iterator());
    SortedNumericSortField sf = (SortedNumericSortField) sortField;
    return Sorter.getDocComparator(maxDoc, sf, () -> null, () -> SortedNumericSelector.wrap(docValues, sf.getSelector(), sf.getNumericType()));
}

Example 15 with SortedNumericSortField

use of org.apache.lucene.search.SortedNumericSortField in project lucene-solr by apache.

the class SimpleTextSegmentInfoFormat method write.

@Override
public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
    String segFileName = IndexFileNames.segmentFileName(si.name, "", SimpleTextSegmentInfoFormat.SI_EXTENSION);
    try (IndexOutput output = dir.createOutput(segFileName, ioContext)) {
        // Only add the file once we've successfully created it, else IFD assert can trip:
        si.addFile(segFileName);
        BytesRefBuilder scratch = new BytesRefBuilder();
        SimpleTextUtil.write(output, SI_VERSION);
        SimpleTextUtil.write(output, si.getVersion().toString(), scratch);
        SimpleTextUtil.writeNewline(output);
        SimpleTextUtil.write(output, SI_MIN_VERSION);
        if (si.getMinVersion() == null) {
            SimpleTextUtil.write(output, "null", scratch);
        } else {
            SimpleTextUtil.write(output, si.getMinVersion().toString(), scratch);
        }
        SimpleTextUtil.writeNewline(output);
        SimpleTextUtil.write(output, SI_DOCCOUNT);
        SimpleTextUtil.write(output, Integer.toString(si.maxDoc()), scratch);
        SimpleTextUtil.writeNewline(output);
        SimpleTextUtil.write(output, SI_USECOMPOUND);
        SimpleTextUtil.write(output, Boolean.toString(si.getUseCompoundFile()), scratch);
        SimpleTextUtil.writeNewline(output);
        Map<String, String> diagnostics = si.getDiagnostics();
        int numDiagnostics = diagnostics == null ? 0 : diagnostics.size();
        SimpleTextUtil.write(output, SI_NUM_DIAG);
        SimpleTextUtil.write(output, Integer.toString(numDiagnostics), scratch);
        SimpleTextUtil.writeNewline(output);
        if (numDiagnostics > 0) {
            for (Map.Entry<String, String> diagEntry : diagnostics.entrySet()) {
                SimpleTextUtil.write(output, SI_DIAG_KEY);
                SimpleTextUtil.write(output, diagEntry.getKey(), scratch);
                SimpleTextUtil.writeNewline(output);
                SimpleTextUtil.write(output, SI_DIAG_VALUE);
                SimpleTextUtil.write(output, diagEntry.getValue(), scratch);
                SimpleTextUtil.writeNewline(output);
            }
        }
        Map<String, String> attributes = si.getAttributes();
        SimpleTextUtil.write(output, SI_NUM_ATT);
        SimpleTextUtil.write(output, Integer.toString(attributes.size()), scratch);
        SimpleTextUtil.writeNewline(output);
        for (Map.Entry<String, String> attEntry : attributes.entrySet()) {
            SimpleTextUtil.write(output, SI_ATT_KEY);
            SimpleTextUtil.write(output, attEntry.getKey(), scratch);
            SimpleTextUtil.writeNewline(output);
            SimpleTextUtil.write(output, SI_ATT_VALUE);
            SimpleTextUtil.write(output, attEntry.getValue(), scratch);
            SimpleTextUtil.writeNewline(output);
        }
        Set<String> files = si.files();
        int numFiles = files == null ? 0 : files.size();
        SimpleTextUtil.write(output, SI_NUM_FILES);
        SimpleTextUtil.write(output, Integer.toString(numFiles), scratch);
        SimpleTextUtil.writeNewline(output);
        if (numFiles > 0) {
            for (String fileName : files) {
                SimpleTextUtil.write(output, SI_FILE);
                SimpleTextUtil.write(output, fileName, scratch);
                SimpleTextUtil.writeNewline(output);
            }
        }
        SimpleTextUtil.write(output, SI_ID);
        SimpleTextUtil.write(output, new BytesRef(si.getId()));
        SimpleTextUtil.writeNewline(output);
        Sort indexSort = si.getIndexSort();
        SimpleTextUtil.write(output, SI_SORT);
        final int numSortFields = indexSort == null ? 0 : indexSort.getSort().length;
        SimpleTextUtil.write(output, Integer.toString(numSortFields), scratch);
        SimpleTextUtil.writeNewline(output);
        for (int i = 0; i < numSortFields; ++i) {
            final SortField sortField = indexSort.getSort()[i];
            SimpleTextUtil.write(output, SI_SORT_FIELD);
            SimpleTextUtil.write(output, sortField.getField(), scratch);
            SimpleTextUtil.writeNewline(output);
            SimpleTextUtil.write(output, SI_SORT_TYPE);
            final String sortTypeString;
            final SortField.Type sortType;
            final boolean multiValued;
            if (sortField instanceof SortedSetSortField) {
                sortType = SortField.Type.STRING;
                multiValued = true;
            } else if (sortField instanceof SortedNumericSortField) {
                sortType = ((SortedNumericSortField) sortField).getNumericType();
                multiValued = true;
            } else {
                sortType = sortField.getType();
                multiValued = false;
            }
            switch(sortType) {
                case STRING:
                    if (multiValued) {
                        sortTypeString = "multi_valued_string";
                    } else {
                        sortTypeString = "string";
                    }
                    break;
                case LONG:
                    if (multiValued) {
                        sortTypeString = "multi_valued_long";
                    } else {
                        sortTypeString = "long";
                    }
                    break;
                case INT:
                    if (multiValued) {
                        sortTypeString = "multi_valued_int";
                    } else {
                        sortTypeString = "int";
                    }
                    break;
                case DOUBLE:
                    if (multiValued) {
                        sortTypeString = "multi_valued_double";
                    } else {
                        sortTypeString = "double";
                    }
                    break;
                case FLOAT:
                    if (multiValued) {
                        sortTypeString = "multi_valued_float";
                    } else {
                        sortTypeString = "float";
                    }
                    break;
                default:
                    throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
            }
            SimpleTextUtil.write(output, sortTypeString, scratch);
            SimpleTextUtil.writeNewline(output);
            if (sortField instanceof SortedSetSortField) {
                SortedSetSelector.Type selector = ((SortedSetSortField) sortField).getSelector();
                final String selectorString;
                if (selector == SortedSetSelector.Type.MIN) {
                    selectorString = "min";
                } else if (selector == SortedSetSelector.Type.MIDDLE_MIN) {
                    selectorString = "middle_min";
                } else if (selector == SortedSetSelector.Type.MIDDLE_MAX) {
                    selectorString = "middle_max";
                } else if (selector == SortedSetSelector.Type.MAX) {
                    selectorString = "max";
                } else {
                    throw new IllegalStateException("Unexpected SortedSetSelector type selector: " + selector);
                }
                SimpleTextUtil.write(output, SI_SELECTOR_TYPE);
                SimpleTextUtil.write(output, selectorString, scratch);
                SimpleTextUtil.writeNewline(output);
            } else if (sortField instanceof SortedNumericSortField) {
                SortedNumericSelector.Type selector = ((SortedNumericSortField) sortField).getSelector();
                final String selectorString;
                if (selector == SortedNumericSelector.Type.MIN) {
                    selectorString = "min";
                } else if (selector == SortedNumericSelector.Type.MAX) {
                    selectorString = "max";
                } else {
                    throw new IllegalStateException("Unexpected SortedNumericSelector type selector: " + selector);
                }
                SimpleTextUtil.write(output, SI_SELECTOR_TYPE);
                SimpleTextUtil.write(output, selectorString, scratch);
                SimpleTextUtil.writeNewline(output);
            }
            SimpleTextUtil.write(output, SI_SORT_REVERSE);
            SimpleTextUtil.write(output, Boolean.toString(sortField.getReverse()), scratch);
            SimpleTextUtil.writeNewline(output);
            SimpleTextUtil.write(output, SI_SORT_MISSING);
            final Object missingValue = sortField.getMissingValue();
            final String missing;
            if (missingValue == null) {
                missing = "null";
            } else if (missingValue == SortField.STRING_FIRST) {
                missing = "first";
            } else if (missingValue == SortField.STRING_LAST) {
                missing = "last";
            } else {
                missing = missingValue.toString();
            }
            SimpleTextUtil.write(output, missing, scratch);
            SimpleTextUtil.writeNewline(output);
        }
        SimpleTextUtil.writeChecksum(output, scratch);
    }
}