Examples with StoredField org.apache.lucene.document.StoredField used on opensource projects

Example 21 with StoredField

use of org.apache.lucene.document.StoredField in project lucene-solr by apache.

the class BBoxStrategy method createIndexableFields.

private Field[] createIndexableFields(Rectangle bbox) {
    Field[] fields = new Field[fieldsLen];
    int idx = -1;
    if (hasStored) {
        fields[++idx] = new StoredField(field_minX, bbox.getMinX());
        fields[++idx] = new StoredField(field_minY, bbox.getMinY());
        fields[++idx] = new StoredField(field_maxX, bbox.getMaxX());
        fields[++idx] = new StoredField(field_maxY, bbox.getMaxY());
    }
    if (hasDocVals) {
        fields[++idx] = new DoubleDocValuesField(field_minX, bbox.getMinX());
        fields[++idx] = new DoubleDocValuesField(field_minY, bbox.getMinY());
        fields[++idx] = new DoubleDocValuesField(field_maxX, bbox.getMaxX());
        fields[++idx] = new DoubleDocValuesField(field_maxY, bbox.getMaxY());
    }
    if (hasPointVals) {
        fields[++idx] = new DoublePoint(field_minX, bbox.getMinX());
        fields[++idx] = new DoublePoint(field_minY, bbox.getMinY());
        fields[++idx] = new DoublePoint(field_maxX, bbox.getMaxX());
        fields[++idx] = new DoublePoint(field_maxY, bbox.getMaxY());
    }
    if (xdlFieldType != null) {
        fields[++idx] = new Field(field_xdl, bbox.getCrossesDateLine() ? "T" : "F", xdlFieldType);
    }
    assert idx == fields.length - 1;
    return fields;
}

Example 22 with StoredField

use of org.apache.lucene.document.StoredField in project lucene-solr by apache.

the class DocumentDictionaryTest method indexMultiValuedDocuments.

private List<Suggestion> indexMultiValuedDocuments(int numDocs, RandomIndexWriter writer) throws IOException {
    List<Suggestion> suggestionList = new ArrayList<>(numDocs);
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        Field field;
        BytesRef payloadValue;
        Set<BytesRef> contextValues = new HashSet<>();
        //-1 for missing weight
        long numericValue = -1;
        BytesRef term;
        payloadValue = new BytesRef("payload_" + i);
        field = new StoredField(PAYLOAD_FIELD_NAME, payloadValue);
        doc.add(field);
        if (usually()) {
            numericValue = 100 + i;
            field = new NumericDocValuesField(WEIGHT_FIELD_NAME, numericValue);
            doc.add(field);
        }
        int numContexts = atLeast(1);
        for (int j = 0; j < numContexts; j++) {
            BytesRef contextValue = new BytesRef("context_" + i + "_" + j);
            field = new StoredField(CONTEXT_FIELD_NAME, contextValue);
            doc.add(field);
            contextValues.add(contextValue);
        }
        int numSuggestions = atLeast(2);
        for (int j = 0; j < numSuggestions; j++) {
            term = new BytesRef("field_" + i + "_" + j);
            field = new StoredField(FIELD_NAME, term);
            doc.add(field);
            Suggestion suggestionValue = new Suggestion();
            suggestionValue.payload = payloadValue;
            suggestionValue.contexts = contextValues;
            suggestionValue.weight = numericValue;
            suggestionValue.term = term;
            suggestionList.add(suggestionValue);
        }
        writer.addDocument(doc);
    }
    return suggestionList;
}

Example 23 with StoredField

use of org.apache.lucene.document.StoredField in project lucene-solr by apache.

the class DocumentDictionaryTest method generateIndexDocuments.

/** Returns Pair(list of invalid document terms, Map of document term -> document) */
private Map.Entry<List<String>, Map<String, Document>> generateIndexDocuments(int ndocs, boolean requiresContexts) {
    Map<String, Document> docs = new HashMap<>();
    List<String> invalidDocTerms = new ArrayList<>();
    for (int i = 0; i < ndocs; i++) {
        Document doc = new Document();
        boolean invalidDoc = false;
        Field field = null;
        // usually have valid term field in document
        if (usually()) {
            field = new TextField(FIELD_NAME, "field_" + i, Field.Store.YES);
            doc.add(field);
        } else {
            invalidDoc = true;
        }
        // even if payload is not required usually have it
        if (usually()) {
            Field payload = new StoredField(PAYLOAD_FIELD_NAME, new BytesRef("payload_" + i));
            doc.add(payload);
        }
        if (requiresContexts || usually()) {
            if (usually()) {
                for (int j = 0; j < atLeast(2); j++) {
                    doc.add(new StoredField(CONTEXT_FIELD_NAME, new BytesRef("context_" + i + "_" + j)));
                }
            }
        // we should allow entries without context
        }
        // usually have valid weight field in document
        if (usually()) {
            Field weight = (rarely()) ? new StoredField(WEIGHT_FIELD_NAME, 100d + i) : new NumericDocValuesField(WEIGHT_FIELD_NAME, 100 + i);
            doc.add(weight);
        }
        String term = null;
        if (invalidDoc) {
            term = (field != null) ? field.stringValue() : "invalid_" + i;
            invalidDocTerms.add(term);
        } else {
            term = field.stringValue();
        }
        docs.put(term, doc);
    }
    return new SimpleEntry<>(invalidDocTerms, docs);
}

Example 24 with StoredField

use of org.apache.lucene.document.StoredField in project lucene-solr by apache.

the class BaseStoredFieldsFormatTestCase method testNumericField.

public void testNumericField() throws Exception {
    Directory dir = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(500);
    final Number[] answers = new Number[numDocs];
    final Class<?>[] typeAnswers = new Class<?>[numDocs];
    for (int id = 0; id < numDocs; id++) {
        Document doc = new Document();
        final Field nf;
        final Number answer;
        final Class<?> typeAnswer;
        if (random().nextBoolean()) {
            // float/double
            if (random().nextBoolean()) {
                final float f = random().nextFloat();
                answer = Float.valueOf(f);
                nf = new StoredField("nf", f);
                typeAnswer = Float.class;
            } else {
                final double d = random().nextDouble();
                answer = Double.valueOf(d);
                nf = new StoredField("nf", d);
                typeAnswer = Double.class;
            }
        } else {
            // int/long
            if (random().nextBoolean()) {
                final int i = random().nextInt();
                answer = Integer.valueOf(i);
                nf = new StoredField("nf", i);
                typeAnswer = Integer.class;
            } else {
                final long l = random().nextLong();
                answer = Long.valueOf(l);
                nf = new StoredField("nf", l);
                typeAnswer = Long.class;
            }
        }
        doc.add(nf);
        answers[id] = answer;
        typeAnswers[id] = typeAnswer;
        doc.add(new StoredField("id", id));
        doc.add(new IntPoint("id", id));
        doc.add(new NumericDocValuesField("id", id));
        w.addDocument(doc);
    }
    final DirectoryReader r = w.getReader();
    w.close();
    assertEquals(numDocs, r.numDocs());
    for (LeafReaderContext ctx : r.leaves()) {
        final LeafReader sub = ctx.reader();
        final NumericDocValues ids = DocValues.getNumeric(sub, "id");
        for (int docID = 0; docID < sub.numDocs(); docID++) {
            final Document doc = sub.document(docID);
            final Field f = (Field) doc.getField("nf");
            assertTrue("got f=" + f, f instanceof StoredField);
            assertEquals(docID, ids.nextDoc());
            assertEquals(answers[(int) ids.longValue()], f.numericValue());
        }
    }
    r.close();
    dir.close();
}

Example 25 with StoredField

use of org.apache.lucene.document.StoredField in project lucene-solr by apache.

the class BaseStoredFieldsFormatTestCase method testWriteReadMerge.

public void testWriteReadMerge() throws IOException {
    // get another codec, other than the default: so we are merging segments across different codecs
    final Codec otherCodec;
    if ("SimpleText".equals(Codec.getDefault().getName())) {
        otherCodec = TestUtil.getDefaultCodec();
    } else {
        otherCodec = new SimpleTextCodec();
    }
    Directory dir = newDirectory();
    IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
    final int docCount = atLeast(200);
    final byte[][][] data = new byte[docCount][][];
    for (int i = 0; i < docCount; ++i) {
        final int fieldCount = rarely() ? RandomNumbers.randomIntBetween(random(), 1, 500) : RandomNumbers.randomIntBetween(random(), 1, 5);
        data[i] = new byte[fieldCount][];
        for (int j = 0; j < fieldCount; ++j) {
            final int length = rarely() ? random().nextInt(1000) : random().nextInt(10);
            final int max = rarely() ? 256 : 2;
            data[i][j] = randomByteArray(length, max);
        }
    }
    final FieldType type = new FieldType(StringField.TYPE_STORED);
    type.setIndexOptions(IndexOptions.NONE);
    type.freeze();
    IntPoint id = new IntPoint("id", 0);
    StoredField idStored = new StoredField("id", 0);
    for (int i = 0; i < data.length; ++i) {
        Document doc = new Document();
        doc.add(id);
        doc.add(idStored);
        id.setIntValue(i);
        idStored.setIntValue(i);
        for (int j = 0; j < data[i].length; ++j) {
            Field f = new Field("bytes" + j, data[i][j], type);
            doc.add(f);
        }
        iw.w.addDocument(doc);
        if (random().nextBoolean() && (i % (data.length / 10) == 0)) {
            iw.w.close();
            IndexWriterConfig iwConfNew = newIndexWriterConfig(new MockAnalyzer(random()));
            // test merging against a non-compressing codec
            if (iwConf.getCodec() == otherCodec) {
                iwConfNew.setCodec(Codec.getDefault());
            } else {
                iwConfNew.setCodec(otherCodec);
            }
            iwConf = iwConfNew;
            iw = new RandomIndexWriter(random(), dir, iwConf);
        }
    }
    for (int i = 0; i < 10; ++i) {
        final int min = random().nextInt(data.length);
        final int max = min + random().nextInt(20);
        iw.deleteDocuments(IntPoint.newRangeQuery("id", min, max - 1));
    }
    // force merges with deletions
    iw.forceMerge(2);
    iw.commit();
    final DirectoryReader ir = DirectoryReader.open(dir);
    assertTrue(ir.numDocs() > 0);
    int numDocs = 0;
    for (int i = 0; i < ir.maxDoc(); ++i) {
        final Document doc = ir.document(i);
        if (doc == null) {
            continue;
        }
        ++numDocs;
        final int docId = doc.getField("id").numericValue().intValue();
        assertEquals(data[docId].length + 1, doc.getFields().size());
        for (int j = 0; j < data[docId].length; ++j) {
            final byte[] arr = data[docId][j];
            final BytesRef arr2Ref = doc.getBinaryValue("bytes" + j);
            final byte[] arr2 = Arrays.copyOfRange(arr2Ref.bytes, arr2Ref.offset, arr2Ref.offset + arr2Ref.length);
            assertArrayEquals(arr, arr2);
        }
    }
    assertTrue(ir.numDocs() <= numDocs);
    ir.close();
    iw.deleteAll();
    iw.commit();
    iw.forceMerge(1);
    iw.close();
    dir.close();
}