Examples with IndexOutput org.apache.lucene.store.IndexOutput used on opensource projects

Example 96 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class CompletionFieldsConsumer method close.

@Override
public void close() throws IOException {
    if (closed) {
        return;
    }
    closed = true;
    String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, INDEX_EXTENSION);
    boolean success = false;
    try (IndexOutput indexOut = state.directory.createOutput(indexFile, state.context)) {
        delegateFieldsConsumer.close();
        CodecUtil.writeIndexHeader(indexOut, CODEC_NAME, COMPLETION_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
        /*
       * we write the delegate postings format name so we can load it
       * without getting an instance in the ctor
       */
        indexOut.writeString(delegatePostingsFormatName);
        // write # of seen fields
        indexOut.writeVInt(seenFields.size());
        // write field numbers and dictOut offsets
        for (Map.Entry<String, CompletionMetaData> seenField : seenFields.entrySet()) {
            FieldInfo fieldInfo = state.fieldInfos.fieldInfo(seenField.getKey());
            indexOut.writeVInt(fieldInfo.number);
            CompletionMetaData metaData = seenField.getValue();
            indexOut.writeVLong(metaData.filePointer);
            indexOut.writeVLong(metaData.minWeight);
            indexOut.writeVLong(metaData.maxWeight);
            indexOut.writeByte(metaData.type);
        }
        CodecUtil.writeFooter(indexOut);
        CodecUtil.writeFooter(dictOut);
        IOUtils.close(dictOut);
        success = true;
    } finally {
        if (success == false) {
            IOUtils.closeWhileHandlingException(dictOut, delegateFieldsConsumer);
        }
    }
}

Example 97 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestIndexInput method testRawIndexInputRead.

// this test checks the IndexInput methods of any impl
public void testRawIndexInputRead() throws IOException {
    for (int i = 0; i < 10; i++) {
        Random random = random();
        final Directory dir = newDirectory();
        IndexOutput os = dir.createOutput("foo", newIOContext(random));
        os.writeBytes(READ_TEST_BYTES, READ_TEST_BYTES.length);
        os.close();
        IndexInput is = dir.openInput("foo", newIOContext(random));
        checkReads(is, IOException.class);
        is.close();
        os = dir.createOutput("bar", newIOContext(random));
        os.writeBytes(RANDOM_TEST_BYTES, RANDOM_TEST_BYTES.length);
        os.close();
        is = dir.openInput("bar", newIOContext(random));
        checkRandomReads(is);
        is.close();
        dir.close();
    }
}

Example 98 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestIndexFileDeleter method copyFile.

public void copyFile(Directory dir, String src, String dest) throws IOException {
    IndexInput in = dir.openInput(src, newIOContext(random()));
    IndexOutput out = dir.createOutput(dest, newIOContext(random()));
    byte[] b = new byte[1024];
    long remainder = in.length();
    while (remainder > 0) {
        int len = (int) Math.min(b.length, remainder);
        in.readBytes(b, 0, len);
        out.writeBytes(b, len);
        remainder -= len;
    }
    in.close();
    out.close();
}

Example 99 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class Test2BPagedBytes method test.

public void test() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("test2BPagedBytes"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    PagedBytes pb = new PagedBytes(15);
    IndexOutput dataOutput = dir.createOutput("foo", IOContext.DEFAULT);
    long netBytes = 0;
    long seed = random().nextLong();
    long lastFP = 0;
    Random r2 = new Random(seed);
    while (netBytes < 1.1 * Integer.MAX_VALUE) {
        int numBytes = TestUtil.nextInt(r2, 1, 32768);
        byte[] bytes = new byte[numBytes];
        r2.nextBytes(bytes);
        dataOutput.writeBytes(bytes, bytes.length);
        long fp = dataOutput.getFilePointer();
        assert fp == lastFP + numBytes;
        lastFP = fp;
        netBytes += numBytes;
    }
    dataOutput.close();
    IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
    pb.copy(input, input.length());
    input.close();
    PagedBytes.Reader reader = pb.freeze(true);
    r2 = new Random(seed);
    netBytes = 0;
    while (netBytes < 1.1 * Integer.MAX_VALUE) {
        int numBytes = TestUtil.nextInt(r2, 1, 32768);
        byte[] bytes = new byte[numBytes];
        r2.nextBytes(bytes);
        BytesRef expected = new BytesRef(bytes);
        BytesRef actual = new BytesRef();
        reader.fillSlice(actual, netBytes, numBytes);
        assertEquals(expected, actual);
        netBytes += numBytes;
    }
    dir.close();
}

Example 100 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class MockRandomPostingsFormat method fieldsConsumer.

@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
    int minSkipInterval;
    if (state.segmentInfo.maxDoc() > 1000000) {
        // Test2BPostings can OOME otherwise:
        minSkipInterval = 3;
    } else {
        minSkipInterval = 2;
    }
    // we pull this before the seed intentionally: because it's not consumed at runtime
    // (the skipInterval is written into postings header).
    // NOTE: Currently not passed to postings writer.
    //       before, it was being passed in wrongly as acceptableOverhead!
    int skipInterval = TestUtil.nextInt(seedRandom, minSkipInterval, 10);
    if (LuceneTestCase.VERBOSE) {
        System.out.println("MockRandomCodec: skipInterval=" + skipInterval);
    }
    final long seed = seedRandom.nextLong();
    if (LuceneTestCase.VERBOSE) {
        System.out.println("MockRandomCodec: writing to seg=" + state.segmentInfo.name + " formatID=" + state.segmentSuffix + " seed=" + seed);
    }
    final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, SEED_EXT);
    try (IndexOutput out = state.directory.createOutput(seedFileName, state.context)) {
        CodecUtil.writeIndexHeader(out, "MockRandomSeed", 0, state.segmentInfo.getId(), state.segmentSuffix);
        out.writeLong(seed);
        CodecUtil.writeFooter(out);
    }
    final Random random = new Random(seed);
    // consume a random for buffersize
    random.nextInt();
    PostingsWriterBase postingsWriter = new Lucene50PostingsWriter(state);
    final FieldsConsumer fields;
    final int t1 = random.nextInt(5);
    if (t1 == 0) {
        boolean success = false;
        try {
            fields = new FSTTermsWriter(state, postingsWriter);
            success = true;
        } finally {
            if (!success) {
                postingsWriter.close();
            }
        }
    } else if (t1 == 1) {
        boolean success = false;
        try {
            fields = new FSTOrdTermsWriter(state, postingsWriter);
            success = true;
        } finally {
            if (!success) {
                postingsWriter.close();
            }
        }
    } else if (t1 == 2) {
        if (LuceneTestCase.VERBOSE) {
            System.out.println("MockRandomCodec: writing BlockTree terms dict");
        }
        // TODO: would be nice to allow 1 but this is very
        // slow to write
        final int minTermsInBlock = TestUtil.nextInt(random, 2, 100);
        final int maxTermsInBlock = Math.max(2, (minTermsInBlock - 1) * 2 + random.nextInt(100));
        boolean success = false;
        try {
            fields = new BlockTreeTermsWriter(state, postingsWriter, minTermsInBlock, maxTermsInBlock);
            success = true;
        } finally {
            if (!success) {
                postingsWriter.close();
            }
        }
    } else if (t1 == 3) {
        if (LuceneTestCase.VERBOSE) {
            System.out.println("MockRandomCodec: writing Block terms dict");
        }
        boolean success = false;
        final TermsIndexWriterBase indexWriter;
        try {
            if (random.nextBoolean()) {
                int termIndexInterval = TestUtil.nextInt(random, 1, 100);
                if (LuceneTestCase.VERBOSE) {
                    System.out.println("MockRandomCodec: fixed-gap terms index (tii=" + termIndexInterval + ")");
                }
                indexWriter = new FixedGapTermsIndexWriter(state, termIndexInterval);
            } else {
                final VariableGapTermsIndexWriter.IndexTermSelector selector;
                final int n2 = random.nextInt(3);
                if (n2 == 0) {
                    final int tii = TestUtil.nextInt(random, 1, 100);
                    selector = new VariableGapTermsIndexWriter.EveryNTermSelector(tii);
                    if (LuceneTestCase.VERBOSE) {
                        System.out.println("MockRandomCodec: variable-gap terms index (tii=" + tii + ")");
                    }
                } else if (n2 == 1) {
                    final int docFreqThresh = TestUtil.nextInt(random, 2, 100);
                    final int tii = TestUtil.nextInt(random, 1, 100);
                    selector = new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThresh, tii);
                } else {
                    final long seed2 = random.nextLong();
                    final int gap = TestUtil.nextInt(random, 2, 40);
                    if (LuceneTestCase.VERBOSE) {
                        System.out.println("MockRandomCodec: random-gap terms index (max gap=" + gap + ")");
                    }
                    selector = new VariableGapTermsIndexWriter.IndexTermSelector() {

                        final Random rand = new Random(seed2);

                        @Override
                        public boolean isIndexTerm(BytesRef term, TermStats stats) {
                            return rand.nextInt(gap) == gap / 2;
                        }

                        @Override
                        public void newField(FieldInfo fieldInfo) {
                        }
                    };
                }
                indexWriter = new VariableGapTermsIndexWriter(state, selector);
            }
            success = true;
        } finally {
            if (!success) {
                postingsWriter.close();
            }
        }
        success = false;
        try {
            fields = new BlockTermsWriter(indexWriter, state, postingsWriter);
            success = true;
        } finally {
            if (!success) {
                try {
                    postingsWriter.close();
                } finally {
                    indexWriter.close();
                }
            }
        }
    } else if (t1 == 4) {
        // Use OrdsBlockTree terms dict
        if (LuceneTestCase.VERBOSE) {
            System.out.println("MockRandomCodec: writing OrdsBlockTree");
        }
        // TODO: would be nice to allow 1 but this is very
        // slow to write
        final int minTermsInBlock = TestUtil.nextInt(random, 2, 100);
        final int maxTermsInBlock = Math.max(2, (minTermsInBlock - 1) * 2 + random.nextInt(100));
        boolean success = false;
        try {
            fields = new OrdsBlockTreeTermsWriter(state, postingsWriter, minTermsInBlock, maxTermsInBlock);
            success = true;
        } finally {
            if (!success) {
                postingsWriter.close();
            }
        }
    } else {
        // BUG!
        throw new AssertionError();
    }
    return fields;
}