Examples with BytesRef org.apache.lucene.util.BytesRef used on opensource projects

Example 81 with BytesRef

use of org.apache.lucene.util.BytesRef in project elasticsearch by elastic.

the class StoreTests method testCheckIntegrity.

public void testCheckIntegrity() throws IOException {
    Directory dir = newDirectory();
    long luceneFileLength = 0;
    try (IndexOutput output = dir.createOutput("lucene_checksum.bin", IOContext.DEFAULT)) {
        int iters = scaledRandomIntBetween(10, 100);
        for (int i = 0; i < iters; i++) {
            BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
            output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
            luceneFileLength += bytesRef.length;
        }
        CodecUtil.writeFooter(output);
        luceneFileLength += CodecUtil.footerLength();
    }
    final long luceneChecksum;
    try (IndexInput indexInput = dir.openInput("lucene_checksum.bin", IOContext.DEFAULT)) {
        assertEquals(luceneFileLength, indexInput.length());
        luceneChecksum = CodecUtil.retrieveChecksum(indexInput);
    }
    dir.close();
}

Example 82 with BytesRef

use of org.apache.lucene.util.BytesRef in project elasticsearch by elastic.

the class StoreTests method testVerifyingIndexInput.

public void testVerifyingIndexInput() throws IOException {
    Directory dir = newDirectory();
    IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
    int iters = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < iters; i++) {
        BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
        output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
    }
    CodecUtil.writeFooter(output);
    output.close();
    // Check file
    IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
    long checksum = CodecUtil.retrieveChecksum(indexInput);
    indexInput.seek(0);
    IndexInput verifyingIndexInput = new Store.VerifyingIndexInput(dir.openInput("foo.bar", IOContext.DEFAULT));
    readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
    Store.verify(verifyingIndexInput);
    assertThat(checksum, equalTo(((ChecksumIndexInput) verifyingIndexInput).getChecksum()));
    IOUtils.close(indexInput, verifyingIndexInput);
    // Corrupt file and check again
    corruptFile(dir, "foo.bar", "foo1.bar");
    verifyingIndexInput = new Store.VerifyingIndexInput(dir.openInput("foo1.bar", IOContext.DEFAULT));
    readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
    try {
        Store.verify(verifyingIndexInput);
        fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
    // ok
    }
    IOUtils.close(verifyingIndexInput);
    IOUtils.close(dir);
}

Example 83 with BytesRef

use of org.apache.lucene.util.BytesRef in project elasticsearch by elastic.

the class FileInfoTests method testInvalidFieldsInFromXContent.

public void testInvalidFieldsInFromXContent() throws IOException {
    final int iters = scaledRandomIntBetween(1, 10);
    for (int iter = 0; iter < iters; iter++) {
        final BytesRef hash = new BytesRef(scaledRandomIntBetween(0, 1024 * 1024));
        hash.length = hash.bytes.length;
        for (int i = 0; i < hash.length; i++) {
            hash.bytes[i] = randomByte();
        }
        String name = "foobar";
        String physicalName = "_foobar";
        String failure = null;
        long length = Math.max(0, Math.abs(randomLong()));
        // random corruption
        switch(randomIntBetween(0, 3)) {
            case 0:
                name = "foo,bar";
                failure = "missing or invalid file name";
                break;
            case 1:
                physicalName = "_foo,bar";
                failure = "missing or invalid physical file name";
                break;
            case 2:
                length = -Math.abs(randomLong());
                failure = "missing or invalid file length";
                break;
            case 3:
                break;
            default:
                fail("shouldn't be here");
        }
        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
        builder.startObject();
        builder.field(FileInfo.NAME, name);
        builder.field(FileInfo.PHYSICAL_NAME, physicalName);
        builder.field(FileInfo.LENGTH, length);
        builder.field(FileInfo.WRITTEN_BY, Version.LATEST.toString());
        builder.field(FileInfo.CHECKSUM, "666");
        builder.endObject();
        byte[] xContent = BytesReference.toBytes(builder.bytes());
        if (failure == null) {
            // No failures should read as usual
            final BlobStoreIndexShardSnapshot.FileInfo parsedInfo;
            try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
                parser.nextToken();
                parsedInfo = BlobStoreIndexShardSnapshot.FileInfo.fromXContent(parser);
            }
            assertThat(name, equalTo(parsedInfo.name()));
            assertThat(physicalName, equalTo(parsedInfo.physicalName()));
            assertThat(length, equalTo(parsedInfo.length()));
            assertEquals("666", parsedInfo.checksum());
            assertEquals("666", parsedInfo.metadata().checksum());
            assertEquals(Version.LATEST, parsedInfo.metadata().writtenBy());
        } else {
            try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
                parser.nextToken();
                BlobStoreIndexShardSnapshot.FileInfo.fromXContent(parser);
                fail("Should have failed with [" + failure + "]");
            } catch (ElasticsearchParseException ex) {
                assertThat(ex.getMessage(), containsString(failure));
            }
        }
    }
}

Example 84 with BytesRef

use of org.apache.lucene.util.BytesRef in project elasticsearch by elastic.

the class StoreTests method testChecksumCorrupted.

public void testChecksumCorrupted() throws IOException {
    Directory dir = newDirectory();
    IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
    int iters = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < iters; i++) {
        BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
        output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
    }
    output.writeInt(CodecUtil.FOOTER_MAGIC);
    output.writeInt(0);
    String checksum = Store.digestToString(output.getChecksum());
    // write a wrong checksum to the file
    output.writeLong(output.getChecksum() + 1);
    output.close();
    IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
    indexInput.seek(0);
    BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
    long length = indexInput.length();
    IndexOutput verifyingOutput = new Store.LuceneVerifyingIndexOutput(new StoreFileMetaData("foo1.bar", length, checksum), dir.createOutput("foo1.bar", IOContext.DEFAULT));
    // we write the checksum in the try / catch block below
    length -= 8;
    while (length > 0) {
        if (random().nextInt(10) == 0) {
            verifyingOutput.writeByte(indexInput.readByte());
            length--;
        } else {
            int min = (int) Math.min(length, ref.bytes.length);
            indexInput.readBytes(ref.bytes, ref.offset, min);
            verifyingOutput.writeBytes(ref.bytes, ref.offset, min);
            length -= min;
        }
    }
    try {
        BytesRef checksumBytes = new BytesRef(8);
        checksumBytes.length = 8;
        indexInput.readBytes(checksumBytes.bytes, checksumBytes.offset, checksumBytes.length);
        if (randomBoolean()) {
            verifyingOutput.writeBytes(checksumBytes.bytes, checksumBytes.offset, checksumBytes.length);
        } else {
            for (int i = 0; i < checksumBytes.length; i++) {
                verifyingOutput.writeByte(checksumBytes.bytes[i]);
            }
        }
        fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
    // ok
    }
    IOUtils.close(indexInput, verifyingOutput, dir);
}

Example 85 with BytesRef

use of org.apache.lucene.util.BytesRef in project elasticsearch by elastic.

the class StoreTests method readIndexInputFullyWithRandomSeeks.

private void readIndexInputFullyWithRandomSeeks(IndexInput indexInput) throws IOException {
    BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
    long pos = 0;
    while (pos < indexInput.length()) {
        assertEquals(pos, indexInput.getFilePointer());
        int op = random().nextInt(5);
        if (op == 0) {
            int shift = 100 - randomIntBetween(0, 200);
            pos = Math.min(indexInput.length() - 1, Math.max(0, pos + shift));
            indexInput.seek(pos);
        } else if (op == 1) {
            indexInput.readByte();
            pos++;
        } else {
            int min = (int) Math.min(indexInput.length() - pos, ref.bytes.length);
            indexInput.readBytes(ref.bytes, ref.offset, min);
            pos += min;
        }
    }
}