Examples with IndexOutput org.apache.lucene.store.IndexOutput used on opensource projects

Example 76 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class InputStreamIndexInputTests method testSingleReadTwoBytesLimit.

public void testSingleReadTwoBytesLimit() throws IOException {
    RAMDirectory dir = new RAMDirectory();
    IndexOutput output = dir.createOutput("test", IOContext.DEFAULT);
    for (int i = 0; i < 3; i++) {
        output.writeByte((byte) 1);
    }
    for (int i = 0; i < 3; i++) {
        output.writeByte((byte) 2);
    }
    output.close();
    IndexInput input = dir.openInput("test", IOContext.DEFAULT);
    assertThat(input.getFilePointer(), lessThan(input.length()));
    InputStreamIndexInput is = new InputStreamIndexInput(input, 2);
    assertThat(is.actualSizeToRead(), equalTo(2L));
    assertThat(is.read(), equalTo(1));
    assertThat(is.read(), equalTo(1));
    assertThat(is.read(), equalTo(-1));
    assertThat(input.getFilePointer(), lessThan(input.length()));
    is = new InputStreamIndexInput(input, 2);
    assertThat(is.actualSizeToRead(), equalTo(2L));
    assertThat(is.read(), equalTo(1));
    assertThat(is.read(), equalTo(2));
    assertThat(is.read(), equalTo(-1));
    assertThat(input.getFilePointer(), lessThan(input.length()));
    is = new InputStreamIndexInput(input, 2);
    assertThat(is.actualSizeToRead(), equalTo(2L));
    assertThat(is.read(), equalTo(2));
    assertThat(is.read(), equalTo(2));
    assertThat(is.read(), equalTo(-1));
    assertThat(input.getFilePointer(), equalTo(input.length()));
    is = new InputStreamIndexInput(input, 2);
    assertThat(is.actualSizeToRead(), equalTo(0L));
    assertThat(is.read(), equalTo(-1));
}

Example 77 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class StoreTests method testMarkCorruptedOnTruncatedSegmentsFile.

public void testMarkCorruptedOnTruncatedSegmentsFile() throws IOException {
    IndexWriterConfig iwc = newIndexWriterConfig();
    final ShardId shardId = new ShardId("index", "_na_", 1);
    DirectoryService directoryService = new LuceneManagedDirectoryService(random());
    Store store = new Store(shardId, INDEX_SETTINGS, directoryService, new DummyShardLock(shardId));
    IndexWriter writer = new IndexWriter(store.directory(), iwc);
    int numDocs = 1 + random().nextInt(10);
    List<Document> docs = new ArrayList<>();
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        doc.add(new StringField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new TextField("body", TestUtil.randomRealisticUnicodeString(random()), random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
        doc.add(new SortedDocValuesField("dv", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
        docs.add(doc);
    }
    for (Document d : docs) {
        writer.addDocument(d);
    }
    writer.commit();
    writer.close();
    SegmentInfos segmentCommitInfos = store.readLastCommittedSegmentsInfo();
    store.directory().deleteFile(segmentCommitInfos.getSegmentsFileName());
    try (IndexOutput out = store.directory().createOutput(segmentCommitInfos.getSegmentsFileName(), IOContext.DEFAULT)) {
    // empty file
    }
    try {
        if (randomBoolean()) {
            store.getMetadata(null);
        } else {
            store.readLastCommittedSegmentsInfo();
        }
        fail("corrupted segments_N file");
    } catch (CorruptIndexException ex) {
    // expected
    }
    assertTrue(store.isMarkedCorrupted());
    // we have to remove the index since it's corrupted and might fail the MocKDirWrapper checkindex call
    Lucene.cleanLuceneIndex(store.directory());
    store.close();
}

Example 78 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class StoreTests method testVerifyingIndexOutput.

public void testVerifyingIndexOutput() throws IOException {
    Directory dir = newDirectory();
    IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
    int iters = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < iters; i++) {
        BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
        output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
    }
    CodecUtil.writeFooter(output);
    output.close();
    IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
    String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput));
    indexInput.seek(0);
    BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
    long length = indexInput.length();
    IndexOutput verifyingOutput = new Store.LuceneVerifyingIndexOutput(new StoreFileMetaData("foo1.bar", length, checksum), dir.createOutput("foo1.bar", IOContext.DEFAULT));
    while (length > 0) {
        if (random().nextInt(10) == 0) {
            verifyingOutput.writeByte(indexInput.readByte());
            length--;
        } else {
            int min = (int) Math.min(length, ref.bytes.length);
            indexInput.readBytes(ref.bytes, ref.offset, min);
            verifyingOutput.writeBytes(ref.bytes, ref.offset, min);
            length -= min;
        }
    }
    Store.verify(verifyingOutput);
    try {
        appendRandomData(verifyingOutput);
        fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
    // ok
    }
    try {
        Store.verify(verifyingOutput);
        fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
    // ok
    }
    IOUtils.close(indexInput, verifyingOutput, dir);
}

Example 79 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class StoreTests method corruptFile.

private void corruptFile(Directory dir, String fileIn, String fileOut) throws IOException {
    IndexInput input = dir.openInput(fileIn, IOContext.READONCE);
    IndexOutput output = dir.createOutput(fileOut, IOContext.DEFAULT);
    long len = input.length();
    byte[] b = new byte[1024];
    long broken = randomInt((int) len - 1);
    long pos = 0;
    while (pos < len) {
        int min = (int) Math.min(input.length() - pos, b.length);
        input.readBytes(b, 0, min);
        if (broken >= pos && broken < pos + min) {
            // Flip one byte
            int flipPos = (int) (broken - pos);
            b[flipPos] = (byte) (b[flipPos] ^ 42);
        }
        output.writeBytes(b, min);
        pos += min;
    }
    IOUtils.close(input, output);
}

Example 80 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class StoreTests method testVerifyingIndexOutputOnEmptyFile.

public void testVerifyingIndexOutputOnEmptyFile() throws IOException {
    Directory dir = newDirectory();
    IndexOutput verifyingOutput = new Store.LuceneVerifyingIndexOutput(new StoreFileMetaData("foo.bar", 0, Store.digestToString(0)), dir.createOutput("foo1.bar", IOContext.DEFAULT));
    try {
        Store.verify(verifyingOutput);
        fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
    // ok
    }
    IOUtils.close(verifyingOutput, dir);
}