Examples with FieldInfo org.apache.lucene.index.FieldInfo used on opensource projects

Example 21 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project elasticsearch-skywalker by jprante.

the class TransportSkywalkerAction method shardOperation.

@Override
protected ShardSkywalkerResponse shardOperation(ShardSkywalkerRequest request) throws ElasticsearchException {
    synchronized (mutex) {
        IndexService indexService = indicesService.indexServiceSafe(request.index());
        InternalIndexShard indexShard = (InternalIndexShard) indexService.shardSafe(request.shardId());
        MapperService mapperService = indexService.mapperService();
        Engine.Searcher searcher = indexShard.acquireSearcher("skywalker_action");
        try {
            IndexReader reader = searcher.reader();
            Skywalker skywalker = new Skywalker(reader);
            Map<String, Object> response = new HashMap();
            Directory directory = indexShard.store().directory();
            List indexFiles = new ArrayList();
            for (String f : skywalker.getIndexFiles(directory)) {
                Map indexFile = new HashMap();
                indexFile.put("name", f);
                indexFile.put("function", skywalker.getFileFunction(f));
                indexFiles.add(indexFile);
            }
            response.put("indexFiles", indexFiles);
            skywalker.getStoreMetadata(response, indexShard.store().getMetadata());
            response.put("indexVersion", skywalker.getVersion());
            response.put("directoryImpl", skywalker.getDirImpl());
            response.put("numDocs", reader.numDocs());
            response.put("maxDoc", reader.maxDoc());
            response.put("hasDeletions", reader.hasDeletions());
            response.put("numDeletedDocs", reader.numDeletedDocs());
            Set<FieldTermCount> ftc = skywalker.getFieldTermCounts();
            response.put("numTerms", skywalker.getNumTerms());
            Map indexFormatInfo = new HashMap();
            FormatDetails details = skywalker.getFormatDetails();
            indexFormatInfo.put("version", details.getVersion());
            indexFormatInfo.put("genericName", details.getGenericName());
            indexFormatInfo.put("capabilities", details.getCapabilities());
            response.put("indexFormat", indexFormatInfo);
            List commits = new ArrayList();
            Iterator<Segment> it = indexShard.engine().segments().iterator();
            while (it.hasNext()) {
                Segment segment = it.next();
                Map m = new HashMap();
                m.put("segment", segment.getName());
                m.put("count", segment.getNumDocs());
                m.put("deleted", segment.getDeletedDocs());
                m.put("generation", segment.getGeneration());
                m.put("sizeInBytes", segment.getSizeInBytes());
                m.put("version", segment.getVersion());
                m.put("committed", segment.committed);
                m.put("compound", segment.compound);
                m.put("size", segment.getSize().toString());
                commits.add(m);
            }
            response.put("commits", commits);
            List fieldInfos = new ArrayList();
            for (FieldInfo fi : MultiFields.getMergedFieldInfos(reader)) {
                fieldInfos.add(skywalker.getFieldInfo(mapperService, fi));
            }
            response.put("fieldInfos", fieldInfos);
            List termList = new ArrayList();
            for (TermStats ts : skywalker.getTopTerms(50)) {
                Map m = new HashMap();
                m.put("field", ts.field());
                m.put("text", ts.text());
                m.put("docFreq", ts.docFreq());
                termList.add(m);
            }
            response.put("topterms", termList);
            return new ShardSkywalkerResponse(request.index(), request.shardId()).setResponse(response);
        } catch (Exception ex) {
            throw new ElasticsearchException(ex.getMessage(), ex);
        }
    }
}

Example 22 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project elasticsearch-skywalker by jprante.

the class DocumentReconstructor method reconstruct.

/**
     * Reconstruct an index shard
     *
     * @return reconstructed document
     * @throws Exception
     */
public XContentBuilder reconstruct(int shardId) throws IOException {
    XContentBuilder builder = jsonBuilder();
    builder.startObject().field("shardId", shardId).field("numDeletions", reader.numDeletedDocs());
    builder.startArray("docs");
    FieldInfos fieldInfos = reader.getFieldInfos();
    Bits live = MultiFields.getLiveDocs(reader);
    for (int docNum = 0; docNum < reader.maxDoc(); docNum++) {
        Document doc = reader.document(docNum);
        if (live != null && live.get(docNum)) {
            // not deleted
            continue;
        }
        builder.startObject().startArray("fields");
        if (fieldInfos != null) {
            for (FieldInfo fi : fieldInfos) {
                String name = fi.name;
                IndexableField[] fs = doc.getFields(name);
                if (fs != null && fs.length > 0) {
                    for (IndexableField f : fs) {
                        IndexableFieldToXContent x = new IndexableFieldToXContent().field(f);
                        x.toXContent(builder, ToXContent.EMPTY_PARAMS);
                    }
                }
            }
        }
        builder.endArray();
        builder.startArray("terms");
        if (fieldInfos != null) {
            TermsEnum te = null;
            DocsAndPositionsEnum dpe = null;
            for (FieldInfo fi : fieldInfos) {
                Terms terms = MultiFields.getTerms(reader, fi.name);
                if (terms == null) {
                    // no terms in this field
                    continue;
                }
                te = terms.iterator(te);
                while (te.next() != null) {
                    DocsAndPositionsEnum newDpe = te.docsAndPositions(live, dpe, 0);
                    if (newDpe == null) {
                        // no position info for this field
                        break;
                    }
                    dpe = newDpe;
                    int num = dpe.advance(docNum);
                    if (num != docNum) {
                        // no data for this term in this doc
                        continue;
                    }
                    String text = te.term().utf8ToString();
                    List<Integer> positions = new ArrayList();
                    List<Integer> starts = new ArrayList();
                    List<Integer> ends = new ArrayList();
                    for (int k = 0; k < dpe.freq(); k++) {
                        int pos = dpe.nextPosition();
                        positions.add(pos);
                        starts.add(dpe.startOffset());
                        ends.add(dpe.endOffset());
                    }
                    builder.startObject().field("text", text).field("positions", positions).field("starts", starts).field("ends", ends).field("count", dpe.freq()).endObject();
                }
            }
        }
        builder.endArray();
        builder.endObject();
    }
    builder.endArray();
    builder.endObject();
    return builder;
}

Example 23 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project lucene-solr by apache.

the class TestInPlaceUpdatesStandalone method testUpdateOfNonExistentDVsShouldNotFail.

@Test
public void testUpdateOfNonExistentDVsShouldNotFail() throws Exception {
    // schema sanity check: assert that the nonexistent_field_i_dvo doesn't exist already
    FieldInfo fi;
    RefCounted<SolrIndexSearcher> holder = h.getCore().getSearcher();
    try {
        fi = holder.get().getSlowAtomicReader().getFieldInfos().fieldInfo("nonexistent_field_i_dvo");
    } finally {
        holder.decref();
    }
    assertNull(fi);
    // Partial update
    addAndGetVersion(sdoc("id", "0", "nonexistent_field_i_dvo", map("set", "42")), null);
    addAndGetVersion(sdoc("id", "1"), null);
    addAndGetVersion(sdoc("id", "1", "nonexistent_field_i_dvo", map("inc", "1")), null);
    addAndGetVersion(sdoc("id", "1", "nonexistent_field_i_dvo", map("inc", "1")), null);
    assertU(commit());
    assertQ(req("q", "*:*"), "//*[@numFound='2']");
    assertQ(req("q", "nonexistent_field_i_dvo:42"), "//*[@numFound='1']");
    assertQ(req("q", "nonexistent_field_i_dvo:2"), "//*[@numFound='1']");
}

Example 24 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project lucene-solr by apache.

the class TestUninvertingReader method testFieldInfos.

public void testFieldInfos() throws IOException {
    Directory dir = newDirectory();
    IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
    Document doc = new Document();
    BytesRef idBytes = new BytesRef("id");
    doc.add(new StringField("id", idBytes, Store.YES));
    doc.add(new LegacyIntField("int", 5, Store.YES));
    doc.add(new NumericDocValuesField("dv", 5));
    doc.add(new IntPoint("dint", 5));
    // not indexed
    doc.add(new StoredField("stored", 5));
    iw.addDocument(doc);
    iw.forceMerge(1);
    iw.close();
    Map<String, Type> uninvertingMap = new HashMap<>();
    uninvertingMap.put("int", Type.LEGACY_INTEGER);
    uninvertingMap.put("dv", Type.LEGACY_INTEGER);
    uninvertingMap.put("dint", Type.INTEGER_POINT);
    DirectoryReader ir = UninvertingReader.wrap(DirectoryReader.open(dir), uninvertingMap);
    LeafReader leafReader = ir.leaves().get(0).reader();
    FieldInfo intFInfo = leafReader.getFieldInfos().fieldInfo("int");
    assertEquals(DocValuesType.NUMERIC, intFInfo.getDocValuesType());
    assertEquals(0, intFInfo.getPointDimensionCount());
    assertEquals(0, intFInfo.getPointNumBytes());
    FieldInfo dintFInfo = leafReader.getFieldInfos().fieldInfo("dint");
    assertEquals(DocValuesType.NUMERIC, dintFInfo.getDocValuesType());
    assertEquals(1, dintFInfo.getPointDimensionCount());
    assertEquals(4, dintFInfo.getPointNumBytes());
    FieldInfo dvFInfo = leafReader.getFieldInfos().fieldInfo("dv");
    assertEquals(DocValuesType.NUMERIC, dvFInfo.getDocValuesType());
    FieldInfo storedFInfo = leafReader.getFieldInfos().fieldInfo("stored");
    assertEquals(DocValuesType.NONE, storedFInfo.getDocValuesType());
    TestUtil.checkReader(ir);
    ir.close();
    dir.close();
}

Example 25 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project lucene-solr by apache.

the class FSTTermsWriter method write.

@Override
public void write(Fields fields) throws IOException {
    for (String field : fields) {
        Terms terms = fields.terms(field);
        if (terms == null) {
            continue;
        }
        FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
        boolean hasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
        TermsEnum termsEnum = terms.iterator();
        TermsWriter termsWriter = new TermsWriter(fieldInfo);
        long sumTotalTermFreq = 0;
        long sumDocFreq = 0;
        FixedBitSet docsSeen = new FixedBitSet(maxDoc);
        while (true) {
            BytesRef term = termsEnum.next();
            if (term == null) {
                break;
            }
            BlockTermState termState = postingsWriter.writeTerm(term, termsEnum, docsSeen);
            if (termState != null) {
                termsWriter.finishTerm(term, termState);
                sumTotalTermFreq += termState.totalTermFreq;
                sumDocFreq += termState.docFreq;
            }
        }
        termsWriter.finish(hasFreq ? sumTotalTermFreq : -1, sumDocFreq, docsSeen.cardinality());
    }
}