Examples with IndexInput org.apache.lucene.store.IndexInput used on opensource projects

Example 61 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class SimpleTextCompoundFormat method getCompoundReader.

@Override
public Directory getCompoundReader(Directory dir, SegmentInfo si, IOContext context) throws IOException {
    String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
    final IndexInput in = dir.openInput(dataFile, context);
    BytesRefBuilder scratch = new BytesRefBuilder();
    // first get to TOC:
    DecimalFormat df = new DecimalFormat(OFFSETPATTERN, DecimalFormatSymbols.getInstance(Locale.ROOT));
    long pos = in.length() - TABLEPOS.length - OFFSETPATTERN.length() - 1;
    in.seek(pos);
    SimpleTextUtil.readLine(in, scratch);
    assert StringHelper.startsWith(scratch.get(), TABLEPOS);
    long tablePos = -1;
    try {
        tablePos = df.parse(stripPrefix(scratch, TABLEPOS)).longValue();
    } catch (ParseException e) {
        throw new CorruptIndexException("can't parse CFS trailer, got: " + scratch.get().utf8ToString(), in);
    }
    // seek to TOC and read it
    in.seek(tablePos);
    SimpleTextUtil.readLine(in, scratch);
    assert StringHelper.startsWith(scratch.get(), TABLE);
    int numEntries = Integer.parseInt(stripPrefix(scratch, TABLE));
    final String[] fileNames = new String[numEntries];
    final long[] startOffsets = new long[numEntries];
    final long[] endOffsets = new long[numEntries];
    for (int i = 0; i < numEntries; i++) {
        SimpleTextUtil.readLine(in, scratch);
        assert StringHelper.startsWith(scratch.get(), TABLENAME);
        fileNames[i] = si.name + IndexFileNames.stripSegmentName(stripPrefix(scratch, TABLENAME));
        if (i > 0) {
            // files must be unique and in sorted order
            assert fileNames[i].compareTo(fileNames[i - 1]) > 0;
        }
        SimpleTextUtil.readLine(in, scratch);
        assert StringHelper.startsWith(scratch.get(), TABLESTART);
        startOffsets[i] = Long.parseLong(stripPrefix(scratch, TABLESTART));
        SimpleTextUtil.readLine(in, scratch);
        assert StringHelper.startsWith(scratch.get(), TABLEEND);
        endOffsets[i] = Long.parseLong(stripPrefix(scratch, TABLEEND));
    }
    return new Directory() {

        private int getIndex(String name) throws IOException {
            int index = Arrays.binarySearch(fileNames, name);
            if (index < 0) {
                throw new FileNotFoundException("No sub-file found (fileName=" + name + " files: " + Arrays.toString(fileNames) + ")");
            }
            return index;
        }

        @Override
        public String[] listAll() throws IOException {
            ensureOpen();
            return fileNames.clone();
        }

        @Override
        public long fileLength(String name) throws IOException {
            ensureOpen();
            int index = getIndex(name);
            return endOffsets[index] - startOffsets[index];
        }

        @Override
        public IndexInput openInput(String name, IOContext context) throws IOException {
            ensureOpen();
            int index = getIndex(name);
            return in.slice(name, startOffsets[index], endOffsets[index] - startOffsets[index]);
        }

        @Override
        public void close() throws IOException {
            in.close();
        }

        // write methods: disabled
        @Override
        public IndexOutput createOutput(String name, IOContext context) {
            throw new UnsupportedOperationException();
        }

        @Override
        public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void sync(Collection<String> names) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void deleteFile(String name) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void rename(String source, String dest) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void syncMetaData() {
            throw new UnsupportedOperationException();
        }

        @Override
        public Lock obtainLock(String name) {
            throw new UnsupportedOperationException();
        }
    };
}

Example 62 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class DirectDocValuesProducer method getMissingBits.

private Bits getMissingBits(FieldInfo field, final long offset, final long length) throws IOException {
    if (offset == -1) {
        return new Bits.MatchAllBits(maxDoc);
    } else {
        FixedBitSet instance;
        synchronized (this) {
            instance = docsWithFieldInstances.get(field.name);
            if (instance == null) {
                IndexInput data = this.data.clone();
                data.seek(offset);
                assert length % 8 == 0;
                long[] bits = new long[(int) length >> 3];
                for (int i = 0; i < bits.length; i++) {
                    bits[i] = data.readLong();
                }
                instance = new FixedBitSet(bits, maxDoc);
                if (!merging) {
                    docsWithFieldInstances.put(field.name, instance);
                    ramBytesUsed.addAndGet(instance.ramBytesUsed());
                }
            }
        }
        return instance;
    }
}

Example 63 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class DirectDocValuesProducer method loadBinary.

private BinaryRawValues loadBinary(BinaryEntry entry) throws IOException {
    IndexInput data = this.data.clone();
    data.seek(entry.offset);
    final byte[] bytes = new byte[entry.numBytes];
    data.readBytes(bytes, 0, entry.numBytes);
    data.seek(entry.offset + entry.numBytes + entry.missingBytes);
    final int[] address = new int[entry.count + 1];
    for (int i = 0; i < entry.count; i++) {
        address[i] = data.readInt();
    }
    address[entry.count] = data.readInt();
    BinaryRawValues values = new BinaryRawValues();
    values.bytes = bytes;
    values.address = address;
    return values;
}

Example 64 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class MemoryDocValuesProducer method getMissingBits.

private Bits getMissingBits(FieldInfo field, final long offset, final long length) throws IOException {
    if (offset == -1) {
        return new Bits.MatchAllBits(maxDoc);
    } else {
        FixedBitSet instance;
        synchronized (this) {
            instance = docsWithFieldInstances.get(field.name);
            if (instance == null) {
                IndexInput data = this.data.clone();
                data.seek(offset);
                assert length % 8 == 0;
                long[] bits = new long[(int) length >> 3];
                for (int i = 0; i < bits.length; i++) {
                    bits[i] = data.readLong();
                }
                instance = new FixedBitSet(bits, maxDoc);
                if (!merging) {
                    docsWithFieldInstances.put(field.name, instance);
                    ramBytesUsed.addAndGet(instance.ramBytesUsed());
                }
            }
        }
        return instance;
    }
}

Example 65 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class MemoryDocValuesProducer method getSortedSet.

@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
    SortedSetEntry sortedSetEntry = sortedSets.get(field.name);
    if (sortedSetEntry.singleton) {
        return DocValues.singleton(getSorted(field));
    }
    final FSTEntry entry = fsts.get(field.name);
    if (entry.numOrds == 0) {
        // empty FST!
        return DocValues.emptySortedSet();
    }
    FST<Long> instance;
    synchronized (this) {
        instance = fstInstances.get(field.name);
        if (instance == null) {
            IndexInput data = this.data.clone();
            data.seek(entry.offset);
            instance = new FST<>(data, PositiveIntOutputs.getSingleton());
            if (!merging) {
                ramBytesUsed.addAndGet(instance.ramBytesUsed());
                fstInstances.put(field.name, instance);
            }
        }
    }
    final LegacyBinaryDocValues docToOrds = getLegacyBinary(field);
    final FST<Long> fst = instance;
    // per-thread resources
    final BytesReader in = fst.getBytesReader();
    final Arc<Long> firstArc = new Arc<>();
    final Arc<Long> scratchArc = new Arc<>();
    final IntsRefBuilder scratchInts = new IntsRefBuilder();
    final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<>(fst);
    final ByteArrayDataInput input = new ByteArrayDataInput();
    return new LegacySortedSetDocValuesWrapper(new LegacySortedSetDocValues() {

        final BytesRefBuilder term = new BytesRefBuilder();

        BytesRef ref;

        long currentOrd;

        @Override
        public long nextOrd() {
            if (input.eof()) {
                return NO_MORE_ORDS;
            } else {
                currentOrd += input.readVLong();
                return currentOrd;
            }
        }

        @Override
        public void setDocument(int docID) {
            ref = docToOrds.get(docID);
            input.reset(ref.bytes, ref.offset, ref.length);
            currentOrd = 0;
        }

        @Override
        public BytesRef lookupOrd(long ord) {
            try {
                in.setPosition(0);
                fst.getFirstArc(firstArc);
                IntsRef output = Util.getByOutput(fst, ord, in, firstArc, scratchArc, scratchInts);
                return Util.toBytesRef(output, term);
            } catch (IOException bogus) {
                throw new RuntimeException(bogus);
            }
        }

        @Override
        public long lookupTerm(BytesRef key) {
            try {
                InputOutput<Long> o = fstEnum.seekCeil(key);
                if (o == null) {
                    return -getValueCount() - 1;
                } else if (o.input.equals(key)) {
                    return o.output.intValue();
                } else {
                    return -o.output - 1;
                }
            } catch (IOException bogus) {
                throw new RuntimeException(bogus);
            }
        }

        @Override
        public long getValueCount() {
            return entry.numOrds;
        }

        @Override
        public TermsEnum termsEnum() {
            return new FSTTermsEnum(fst);
        }
    }, maxDoc);
}