Examples with BytesRefBuilder org.apache.lucene.util.BytesRefBuilder used on opensource projects

Example 21 with BytesRefBuilder

use of org.apache.lucene.util.BytesRefBuilder in project lucene-solr by apache.

the class SimpleTextDocValuesReader method getSorted.

@Override
public SortedDocValues getSorted(FieldInfo fieldInfo) throws IOException {
    final OneField field = fields.get(fieldInfo.name);
    // valid:
    assert field != null;
    final IndexInput in = data.clone();
    final BytesRefBuilder scratch = new BytesRefBuilder();
    final DecimalFormat decoder = new DecimalFormat(field.pattern, new DecimalFormatSymbols(Locale.ROOT));
    final DecimalFormat ordDecoder = new DecimalFormat(field.ordPattern, new DecimalFormatSymbols(Locale.ROOT));
    return new SortedDocValues() {

        int doc = -1;

        @Override
        public int nextDoc() throws IOException {
            return advance(docID() + 1);
        }

        @Override
        public int docID() {
            return doc;
        }

        @Override
        public long cost() {
            return maxDoc;
        }

        int ord;

        @Override
        public int advance(int target) throws IOException {
            for (int i = target; i < maxDoc; ++i) {
                in.seek(field.dataStartFilePointer + field.numValues * (9 + field.pattern.length() + field.maxLength) + i * (1 + field.ordPattern.length()));
                SimpleTextUtil.readLine(in, scratch);
                try {
                    ord = (int) ordDecoder.parse(scratch.get().utf8ToString()).longValue() - 1;
                } catch (ParseException pe) {
                    throw new CorruptIndexException("failed to parse ord", in, pe);
                }
                if (ord >= 0) {
                    return doc = i;
                }
            }
            return doc = NO_MORE_DOCS;
        }

        @Override
        public boolean advanceExact(int target) throws IOException {
            this.doc = target;
            in.seek(field.dataStartFilePointer + field.numValues * (9 + field.pattern.length() + field.maxLength) + target * (1 + field.ordPattern.length()));
            SimpleTextUtil.readLine(in, scratch);
            try {
                ord = (int) ordDecoder.parse(scratch.get().utf8ToString()).longValue() - 1;
            } catch (ParseException pe) {
                throw new CorruptIndexException("failed to parse ord", in, pe);
            }
            return ord >= 0;
        }

        @Override
        public int ordValue() {
            return ord;
        }

        final BytesRefBuilder term = new BytesRefBuilder();

        @Override
        public BytesRef lookupOrd(int ord) throws IOException {
            if (ord < 0 || ord >= field.numValues) {
                throw new IndexOutOfBoundsException("ord must be 0 .. " + (field.numValues - 1) + "; got " + ord);
            }
            in.seek(field.dataStartFilePointer + ord * (9 + field.pattern.length() + field.maxLength));
            SimpleTextUtil.readLine(in, scratch);
            assert StringHelper.startsWith(scratch.get(), LENGTH) : "got " + scratch.get().utf8ToString() + " in=" + in;
            int len;
            try {
                len = decoder.parse(new String(scratch.bytes(), LENGTH.length, scratch.length() - LENGTH.length, StandardCharsets.UTF_8)).intValue();
            } catch (ParseException pe) {
                throw new CorruptIndexException("failed to parse int length", in, pe);
            }
            term.grow(len);
            term.setLength(len);
            in.readBytes(term.bytes(), 0, len);
            return term.get();
        }

        @Override
        public int getValueCount() {
            return (int) field.numValues;
        }
    };
}

Example 22 with BytesRefBuilder

use of org.apache.lucene.util.BytesRefBuilder in project lucene-solr by apache.

the class SimpleTextFieldInfosFormat method write.

@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
    IndexOutput out = directory.createOutput(fileName, context);
    BytesRefBuilder scratch = new BytesRefBuilder();
    boolean success = false;
    try {
        SimpleTextUtil.write(out, NUMFIELDS);
        SimpleTextUtil.write(out, Integer.toString(infos.size()), scratch);
        SimpleTextUtil.writeNewline(out);
        for (FieldInfo fi : infos) {
            SimpleTextUtil.write(out, NAME);
            SimpleTextUtil.write(out, fi.name, scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, NUMBER);
            SimpleTextUtil.write(out, Integer.toString(fi.number), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, INDEXOPTIONS);
            IndexOptions indexOptions = fi.getIndexOptions();
            assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.hasPayloads();
            SimpleTextUtil.write(out, indexOptions.toString(), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, STORETV);
            SimpleTextUtil.write(out, Boolean.toString(fi.hasVectors()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, PAYLOADS);
            SimpleTextUtil.write(out, Boolean.toString(fi.hasPayloads()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, NORMS);
            SimpleTextUtil.write(out, Boolean.toString(!fi.omitsNorms()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, DOCVALUES);
            SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, DOCVALUES_GEN);
            SimpleTextUtil.write(out, Long.toString(fi.getDocValuesGen()), scratch);
            SimpleTextUtil.writeNewline(out);
            Map<String, String> atts = fi.attributes();
            int numAtts = atts == null ? 0 : atts.size();
            SimpleTextUtil.write(out, NUM_ATTS);
            SimpleTextUtil.write(out, Integer.toString(numAtts), scratch);
            SimpleTextUtil.writeNewline(out);
            if (numAtts > 0) {
                for (Map.Entry<String, String> entry : atts.entrySet()) {
                    SimpleTextUtil.write(out, ATT_KEY);
                    SimpleTextUtil.write(out, entry.getKey(), scratch);
                    SimpleTextUtil.writeNewline(out);
                    SimpleTextUtil.write(out, ATT_VALUE);
                    SimpleTextUtil.write(out, entry.getValue(), scratch);
                    SimpleTextUtil.writeNewline(out);
                }
            }
            SimpleTextUtil.write(out, DIM_COUNT);
            SimpleTextUtil.write(out, Integer.toString(fi.getPointDimensionCount()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, DIM_NUM_BYTES);
            SimpleTextUtil.write(out, Integer.toString(fi.getPointNumBytes()), scratch);
            SimpleTextUtil.writeNewline(out);
        }
        SimpleTextUtil.writeChecksum(out, scratch);
        success = true;
    } finally {
        if (success) {
            out.close();
        } else {
            IOUtils.closeWhileHandlingException(out);
        }
    }
}

Example 23 with BytesRefBuilder

use of org.apache.lucene.util.BytesRefBuilder in project lucene-solr by apache.

the class SimpleTextFieldInfosFormat method read.

@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
    ChecksumIndexInput input = directory.openChecksumInput(fileName, iocontext);
    BytesRefBuilder scratch = new BytesRefBuilder();
    boolean success = false;
    try {
        SimpleTextUtil.readLine(input, scratch);
        assert StringHelper.startsWith(scratch.get(), NUMFIELDS);
        final int size = Integer.parseInt(readString(NUMFIELDS.length, scratch));
        FieldInfo[] infos = new FieldInfo[size];
        for (int i = 0; i < size; i++) {
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NAME);
            String name = readString(NAME.length, scratch);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NUMBER);
            int fieldNumber = Integer.parseInt(readString(NUMBER.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), INDEXOPTIONS);
            String s = readString(INDEXOPTIONS.length, scratch);
            final IndexOptions indexOptions = IndexOptions.valueOf(s);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), STORETV);
            boolean storeTermVector = Boolean.parseBoolean(readString(STORETV.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), PAYLOADS);
            boolean storePayloads = Boolean.parseBoolean(readString(PAYLOADS.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NORMS);
            boolean omitNorms = !Boolean.parseBoolean(readString(NORMS.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DOCVALUES);
            String dvType = readString(DOCVALUES.length, scratch);
            final DocValuesType docValuesType = docValuesType(dvType);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DOCVALUES_GEN);
            final long dvGen = Long.parseLong(readString(DOCVALUES_GEN.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NUM_ATTS);
            int numAtts = Integer.parseInt(readString(NUM_ATTS.length, scratch));
            Map<String, String> atts = new HashMap<>();
            for (int j = 0; j < numAtts; j++) {
                SimpleTextUtil.readLine(input, scratch);
                assert StringHelper.startsWith(scratch.get(), ATT_KEY);
                String key = readString(ATT_KEY.length, scratch);
                SimpleTextUtil.readLine(input, scratch);
                assert StringHelper.startsWith(scratch.get(), ATT_VALUE);
                String value = readString(ATT_VALUE.length, scratch);
                atts.put(key, value);
            }
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DIM_COUNT);
            int dimensionalCount = Integer.parseInt(readString(DIM_COUNT.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DIM_NUM_BYTES);
            int dimensionalNumBytes = Integer.parseInt(readString(DIM_NUM_BYTES.length, scratch));
            infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts), dimensionalCount, dimensionalNumBytes);
        }
        SimpleTextUtil.checkFooter(input);
        FieldInfos fieldInfos = new FieldInfos(infos);
        success = true;
        return fieldInfos;
    } finally {
        if (success) {
            input.close();
        } else {
            IOUtils.closeWhileHandlingException(input);
        }
    }
}

Example 24 with BytesRefBuilder

use of org.apache.lucene.util.BytesRefBuilder in project lucene-solr by apache.

the class SimpleTextBKDReader method visitDocIDs.

void visitDocIDs(IndexInput in, long blockFP, IntersectVisitor visitor) throws IOException {
    BytesRefBuilder scratch = new BytesRefBuilder();
    in.seek(blockFP);
    readLine(in, scratch);
    int count = parseInt(scratch, BLOCK_COUNT);
    visitor.grow(count);
    for (int i = 0; i < count; i++) {
        readLine(in, scratch);
        visitor.visit(parseInt(scratch, BLOCK_DOC_ID));
    }
}

Example 25 with BytesRefBuilder

use of org.apache.lucene.util.BytesRefBuilder in project lucene-solr by apache.

the class SimpleTextBKDReader method readDocIDs.

int readDocIDs(IndexInput in, long blockFP, int[] docIDs) throws IOException {
    BytesRefBuilder scratch = new BytesRefBuilder();
    in.seek(blockFP);
    readLine(in, scratch);
    int count = parseInt(scratch, BLOCK_COUNT);
    for (int i = 0; i < count; i++) {
        readLine(in, scratch);
        docIDs[i] = parseInt(scratch, BLOCK_DOC_ID);
    }
    return count;
}