Examples with DoubleDocValuesField org.apache.lucene.document.DoubleDocValuesField used on opensource projects

Example 1 with DoubleDocValuesField

use of org.apache.lucene.document.DoubleDocValuesField in project elasticsearch by elastic.

the class DiversifiedSamplerTests method testDiversifiedSampler.

public void testDiversifiedSampler() throws Exception {
    String[] data = { // "id,cat,name,price,inStock,author_t,series_t,sequence_i,genre_s,genre_id",
    "0553573403,book,A Game of Thrones,7.99,true,George R.R. Martin,A Song of Ice and Fire,1,fantasy,0", "0553579908,book,A Clash of Kings,7.99,true,George R.R. Martin,A Song of Ice and Fire,2,fantasy,0", "055357342X,book,A Storm of Swords,7.99,true,George R.R. Martin,A Song of Ice and Fire,3,fantasy,0", "0553293354,book,Foundation,17.99,true,Isaac Asimov,Foundation Novels,1,scifi,1", "0812521390,book,The Black Company,6.99,false,Glen Cook,The Chronicles of The Black Company,1,fantasy,0", "0812550706,book,Ender's Game,6.99,true,Orson Scott Card,Ender,1,scifi,1", "0441385532,book,Jhereg,7.95,false,Steven Brust,Vlad Taltos,1,fantasy,0", "0380014300,book,Nine Princes In Amber,6.99,true,Roger Zelazny,the Chronicles of Amber,1,fantasy,0", "0805080481,book,The Book of Three,5.99,true,Lloyd Alexander,The Chronicles of Prydain,1,fantasy,0", "080508049X,book,The Black Cauldron,5.99,true,Lloyd Alexander,The Chronicles of Prydain,2,fantasy,0" };
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    for (String entry : data) {
        String[] parts = entry.split(",");
        Document document = new Document();
        document.add(new SortedDocValuesField("id", new BytesRef(parts[0])));
        document.add(new StringField("cat", parts[1], Field.Store.NO));
        document.add(new TextField("name", parts[2], Field.Store.NO));
        document.add(new DoubleDocValuesField("price", Double.valueOf(parts[3])));
        document.add(new StringField("inStock", parts[4], Field.Store.NO));
        document.add(new StringField("author", parts[5], Field.Store.NO));
        document.add(new StringField("series", parts[6], Field.Store.NO));
        document.add(new StringField("sequence", parts[7], Field.Store.NO));
        document.add(new SortedDocValuesField("genre", new BytesRef(parts[8])));
        document.add(new NumericDocValuesField("genre_id", Long.valueOf(parts[9])));
        indexWriter.addDocument(document);
    }
    indexWriter.close();
    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = new IndexSearcher(indexReader);
    MappedFieldType genreFieldType = new KeywordFieldMapper.KeywordFieldType();
    genreFieldType.setName("genre");
    genreFieldType.setHasDocValues(true);
    Consumer<InternalSampler> verify = result -> {
        Terms terms = result.getAggregations().get("terms");
        assertEquals(2, terms.getBuckets().size());
        assertEquals("0805080481", terms.getBuckets().get(0).getKeyAsString());
        assertEquals("0812550706", terms.getBuckets().get(1).getKeyAsString());
    };
    testCase(indexSearcher, genreFieldType, "map", verify);
    testCase(indexSearcher, genreFieldType, "global_ordinals", verify);
    testCase(indexSearcher, genreFieldType, "bytes_hash", verify);
    genreFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    genreFieldType.setName("genre_id");
    testCase(indexSearcher, genreFieldType, null, verify);
    // wrong field:
    genreFieldType = new KeywordFieldMapper.KeywordFieldType();
    genreFieldType.setName("wrong_field");
    genreFieldType.setHasDocValues(true);
    testCase(indexSearcher, genreFieldType, null, result -> {
        Terms terms = result.getAggregations().get("terms");
        assertEquals(1, terms.getBuckets().size());
        assertEquals("0805080481", terms.getBuckets().get(0).getKeyAsString());
    });
    indexReader.close();
    directory.close();
}

Example 2 with DoubleDocValuesField

use of org.apache.lucene.document.DoubleDocValuesField in project querydsl by querydsl.

the class LuceneQueryTest method createDocument.

private Document createDocument(final String docTitle, final String docAuthor, final String docText, final int docYear, final double docGross) {
    Document doc = new Document();
    // Reusing field for performance
    if (titleField == null) {
        titleField = new TextField("title", docTitle, Store.YES);
        doc.add(titleField);
        titleSortedField = new SortedDocValuesField("title", new BytesRef(docTitle));
        doc.add(titleSortedField);
    } else {
        titleField.setStringValue(docTitle);
        titleSortedField.setBytesValue(new BytesRef(docTitle));
        doc.add(titleField);
        doc.add(titleSortedField);
    }
    if (authorField == null) {
        authorField = new TextField("author", docAuthor, Store.YES);
        doc.add(authorField);
        authorSortedField = new SortedDocValuesField("author", new BytesRef(docAuthor));
        doc.add(authorSortedField);
    } else {
        authorField.setStringValue(docAuthor);
        authorSortedField.setBytesValue(new BytesRef(docAuthor));
        doc.add(authorField);
        doc.add(authorSortedField);
    }
    if (textField == null) {
        textField = new TextField("text", docText, Store.YES);
        doc.add(textField);
        textSortedField = new SortedDocValuesField("text", new BytesRef(docText));
        doc.add(textSortedField);
    } else {
        textField.setStringValue(docText);
        textSortedField.setBytesValue(new BytesRef(docText));
        doc.add(textField);
        doc.add(textSortedField);
    }
    if (yearField == null) {
        yearField = new IntField("year", docYear, Store.YES);
        doc.add(yearField);
        yearSortedField = new NumericDocValuesField("year", docYear);
        doc.add(yearSortedField);
    } else {
        yearField.setIntValue(docYear);
        yearSortedField.setLongValue(docYear);
        doc.add(yearField);
        doc.add(yearSortedField);
    }
    if (grossField == null) {
        grossField = new DoubleField("gross", docGross, Store.YES);
        doc.add(grossField);
        grossSortedField = new DoubleDocValuesField("gross", docGross);
        doc.add(grossSortedField);
    } else {
        grossField.setDoubleValue(docGross);
        grossSortedField.setDoubleValue(docGross);
        doc.add(grossField);
        doc.add(grossSortedField);
    }
    return doc;
}

Example 3 with DoubleDocValuesField

use of org.apache.lucene.document.DoubleDocValuesField in project jackrabbit-oak by apache.

the class LuceneDocumentMaker method addTypedOrderedFields.

private boolean addTypedOrderedFields(List<Field> fields, PropertyState property, String pname, PropertyDefinition pd) {
    // Ignore and warn if property multi-valued as not supported
    if (property.getType().isArray()) {
        log.warn("[{}] Ignoring ordered property {} of type {} for path {} as multivalued ordered property not supported", getIndexName(), pname, Type.fromTag(property.getType().tag(), true), path);
        return false;
    }
    int tag = property.getType().tag();
    int idxDefinedTag = pd.getType();
    // Try converting type to the defined type in the index definition
    if (tag != idxDefinedTag) {
        log.debug("[{}] Ordered property defined with type {} differs from property {} with type {} in " + "path {}", getIndexName(), Type.fromTag(idxDefinedTag, false), property.toString(), Type.fromTag(tag, false), path);
        tag = idxDefinedTag;
    }
    String name = FieldNames.createDocValFieldName(pname);
    boolean fieldAdded = false;
    Field f = null;
    try {
        if (tag == Type.LONG.tag()) {
            //TODO Distinguish fields which need to be used for search and for sort
            //If a field is only used for Sort then it can be stored with less precision
            f = new NumericDocValuesField(name, property.getValue(Type.LONG));
        } else if (tag == Type.DATE.tag()) {
            String date = property.getValue(Type.DATE);
            f = new NumericDocValuesField(name, FieldFactory.dateToLong(date));
        } else if (tag == Type.DOUBLE.tag()) {
            f = new DoubleDocValuesField(name, property.getValue(Type.DOUBLE));
        } else if (tag == Type.BOOLEAN.tag()) {
            f = new SortedDocValuesField(name, new BytesRef(property.getValue(Type.BOOLEAN).toString()));
        } else if (tag == Type.STRING.tag()) {
            f = new SortedDocValuesField(name, new BytesRef(property.getValue(Type.STRING)));
        }
        if (f != null) {
            fields.add(f);
            fieldAdded = true;
        }
    } catch (Exception e) {
        log.warn("[{}] Ignoring ordered property. Could not convert property {} of type {} to type {} for path {}", getIndexName(), pname, Type.fromTag(property.getType().tag(), false), Type.fromTag(tag, false), path, e);
    }
    return fieldAdded;
}

Example 4 with DoubleDocValuesField

use of org.apache.lucene.document.DoubleDocValuesField in project lucene-solr by apache.

the class TestBackwardsCompatibility method addDoc.

private void addDoc(IndexWriter writer, int id) throws IOException {
    Document doc = new Document();
    doc.add(new TextField("content", "aaa", Field.Store.NO));
    doc.add(new StringField("id", Integer.toString(id), Field.Store.YES));
    FieldType customType2 = new FieldType(TextField.TYPE_STORED);
    customType2.setStoreTermVectors(true);
    customType2.setStoreTermVectorPositions(true);
    customType2.setStoreTermVectorOffsets(true);
    doc.add(new Field("autf8", "Lu𝄞ce𝅘𝅥𝅮ne  ☠ ab񕰗cd", customType2));
    doc.add(new Field("utf8", "Lu𝄞ce𝅘𝅥𝅮ne  ☠ ab񕰗cd", customType2));
    doc.add(new Field("content2", "here is more content with aaa aaa aaa", customType2));
    doc.add(new Field("fieⱷld", "field with non-ascii name", customType2));
    // add docvalues fields
    doc.add(new NumericDocValuesField("dvByte", (byte) id));
    byte[] bytes = new byte[] { (byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id };
    BytesRef ref = new BytesRef(bytes);
    doc.add(new BinaryDocValuesField("dvBytesDerefFixed", ref));
    doc.add(new BinaryDocValuesField("dvBytesDerefVar", ref));
    doc.add(new SortedDocValuesField("dvBytesSortedFixed", ref));
    doc.add(new SortedDocValuesField("dvBytesSortedVar", ref));
    doc.add(new BinaryDocValuesField("dvBytesStraightFixed", ref));
    doc.add(new BinaryDocValuesField("dvBytesStraightVar", ref));
    doc.add(new DoubleDocValuesField("dvDouble", (double) id));
    doc.add(new FloatDocValuesField("dvFloat", (float) id));
    doc.add(new NumericDocValuesField("dvInt", id));
    doc.add(new NumericDocValuesField("dvLong", id));
    doc.add(new NumericDocValuesField("dvPacked", id));
    doc.add(new NumericDocValuesField("dvShort", (short) id));
    doc.add(new SortedSetDocValuesField("dvSortedSet", ref));
    doc.add(new SortedNumericDocValuesField("dvSortedNumeric", id));
    doc.add(new IntPoint("intPoint1d", id));
    doc.add(new IntPoint("intPoint2d", id, 2 * id));
    doc.add(new FloatPoint("floatPoint1d", (float) id));
    doc.add(new FloatPoint("floatPoint2d", (float) id, (float) 2 * id));
    doc.add(new LongPoint("longPoint1d", id));
    doc.add(new LongPoint("longPoint2d", id, 2 * id));
    doc.add(new DoublePoint("doublePoint1d", (double) id));
    doc.add(new DoublePoint("doublePoint2d", (double) id, (double) 2 * id));
    doc.add(new BinaryPoint("binaryPoint1d", bytes));
    doc.add(new BinaryPoint("binaryPoint2d", bytes, bytes));
    // a field with both offsets and term vectors for a cross-check
    FieldType customType3 = new FieldType(TextField.TYPE_STORED);
    customType3.setStoreTermVectors(true);
    customType3.setStoreTermVectorPositions(true);
    customType3.setStoreTermVectorOffsets(true);
    customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
    doc.add(new Field("content5", "here is more content with aaa aaa aaa", customType3));
    // a field that omits only positions
    FieldType customType4 = new FieldType(TextField.TYPE_STORED);
    customType4.setStoreTermVectors(true);
    customType4.setStoreTermVectorPositions(false);
    customType4.setStoreTermVectorOffsets(true);
    customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
    doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4));
    // TODO: 
    //   index different norms types via similarity (we use a random one currently?!)
    //   remove any analyzer randomness, explicitly add payloads for certain fields.
    writer.addDocument(doc);
}

Example 5 with DoubleDocValuesField

use of org.apache.lucene.document.DoubleDocValuesField in project lucene-solr by apache.

the class TestSearchAfter method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    allSortFields = new ArrayList<>(Arrays.asList(new SortField[] { new SortField("int", SortField.Type.INT, false), new SortField("long", SortField.Type.LONG, false), new SortField("float", SortField.Type.FLOAT, false), new SortField("double", SortField.Type.DOUBLE, false), new SortField("bytes", SortField.Type.STRING, false), new SortField("bytesval", SortField.Type.STRING_VAL, false), new SortField("int", SortField.Type.INT, true), new SortField("long", SortField.Type.LONG, true), new SortField("float", SortField.Type.FLOAT, true), new SortField("double", SortField.Type.DOUBLE, true), new SortField("bytes", SortField.Type.STRING, true), new SortField("bytesval", SortField.Type.STRING_VAL, true), SortField.FIELD_SCORE, SortField.FIELD_DOC }));
    // Also test missing first / last for the "string" sorts:
    for (String field : new String[] { "bytes", "sortedbytesdocvalues" }) {
        for (int rev = 0; rev < 2; rev++) {
            boolean reversed = rev == 0;
            SortField sf = new SortField(field, SortField.Type.STRING, reversed);
            sf.setMissingValue(SortField.STRING_FIRST);
            allSortFields.add(sf);
            sf = new SortField(field, SortField.Type.STRING, reversed);
            sf.setMissingValue(SortField.STRING_LAST);
            allSortFields.add(sf);
        }
    }
    // Also test missing first / last for the "string_val" sorts:
    for (String field : new String[] { "sortedbytesdocvaluesval", "straightbytesdocvalues" }) {
        for (int rev = 0; rev < 2; rev++) {
            boolean reversed = rev == 0;
            SortField sf = new SortField(field, SortField.Type.STRING_VAL, reversed);
            sf.setMissingValue(SortField.STRING_FIRST);
            allSortFields.add(sf);
            sf = new SortField(field, SortField.Type.STRING_VAL, reversed);
            sf.setMissingValue(SortField.STRING_LAST);
            allSortFields.add(sf);
        }
    }
    int limit = allSortFields.size();
    for (int i = 0; i < limit; i++) {
        SortField sf = allSortFields.get(i);
        if (sf.getType() == SortField.Type.INT) {
            SortField sf2 = new SortField(sf.getField(), SortField.Type.INT, sf.getReverse());
            sf2.setMissingValue(random().nextInt());
            allSortFields.add(sf2);
        } else if (sf.getType() == SortField.Type.LONG) {
            SortField sf2 = new SortField(sf.getField(), SortField.Type.LONG, sf.getReverse());
            sf2.setMissingValue(random().nextLong());
            allSortFields.add(sf2);
        } else if (sf.getType() == SortField.Type.FLOAT) {
            SortField sf2 = new SortField(sf.getField(), SortField.Type.FLOAT, sf.getReverse());
            sf2.setMissingValue(random().nextFloat());
            allSortFields.add(sf2);
        } else if (sf.getType() == SortField.Type.DOUBLE) {
            SortField sf2 = new SortField(sf.getField(), SortField.Type.DOUBLE, sf.getReverse());
            sf2.setMissingValue(random().nextDouble());
            allSortFields.add(sf2);
        }
    }
    dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    int numDocs = atLeast(200);
    Random r = random();
    for (int i = 0; i < numDocs; i++) {
        List<Field> fields = new ArrayList<>();
        fields.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
        fields.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
        fields.add(new NumericDocValuesField("byte", (byte) r.nextInt()));
        fields.add(new NumericDocValuesField("short", (short) r.nextInt()));
        fields.add(new NumericDocValuesField("int", r.nextInt()));
        fields.add(new NumericDocValuesField("long", r.nextLong()));
        fields.add(new FloatDocValuesField("float", r.nextFloat()));
        fields.add(new DoubleDocValuesField("double", r.nextDouble()));
        fields.add(new SortedDocValuesField("bytes", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        fields.add(new BinaryDocValuesField("bytesval", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        Document document = new Document();
        document.add(new StoredField("id", "" + i));
        if (VERBOSE) {
            System.out.println("  add doc id=" + i);
        }
        for (Field field : fields) {
            // So we are sometimes missing that field:
            if (random().nextInt(5) != 4) {
                document.add(field);
                if (VERBOSE) {
                    System.out.println("    " + field);
                }
            }
        }
        iw.addDocument(document);
        if (random().nextInt(50) == 17) {
            iw.commit();
        }
    }
    reader = iw.getReader();
    iw.close();
    searcher = newSearcher(reader);
    if (VERBOSE) {
        System.out.println("  searcher=" + searcher);
    }
}