Examples with IndexOutput org.apache.lucene.store.IndexOutput used on opensource projects

Example 1 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class RecoveryTarget method openAndPutIndexOutput.

/**
     * Creates an {@link org.apache.lucene.store.IndexOutput} for the given file name. Note that the
     * IndexOutput actually point at a temporary file.
     * <p>
     * Note: You can use {@link #getOpenIndexOutput(String)} with the same filename to retrieve the same IndexOutput
     * at a later stage
     */
public IndexOutput openAndPutIndexOutput(String fileName, StoreFileMetaData metaData, Store store) throws IOException {
    ensureRefCount();
    String tempFileName = getTempNameForFile(fileName);
    if (tempFileNames.containsKey(tempFileName)) {
        throw new IllegalStateException("output for file [" + fileName + "] has already been created");
    }
    // add first, before it's created
    tempFileNames.put(tempFileName, fileName);
    IndexOutput indexOutput = store.createVerifyingOutput(tempFileName, metaData, IOContext.DEFAULT);
    openIndexOutputs.put(fileName, indexOutput);
    return indexOutput;
}

Example 2 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class RecoveryTarget method writeFileChunk.

@Override
public void writeFileChunk(StoreFileMetaData fileMetaData, long position, BytesReference content, boolean lastChunk, int totalTranslogOps) throws IOException {
    final Store store = store();
    final String name = fileMetaData.name();
    state().getTranslog().totalOperations(totalTranslogOps);
    final RecoveryState.Index indexState = state().getIndex();
    IndexOutput indexOutput;
    if (position == 0) {
        indexOutput = openAndPutIndexOutput(name, fileMetaData, store);
    } else {
        indexOutput = getOpenIndexOutput(name);
    }
    BytesRefIterator iterator = content.iterator();
    BytesRef scratch;
    while ((scratch = iterator.next()) != null) {
        // we iterate over all pages - this is a 0-copy for all core impls
        indexOutput.writeBytes(scratch.bytes, scratch.offset, scratch.length);
    }
    indexState.addRecoveredBytesToFile(name, content.length());
    if (indexOutput.getFilePointer() >= fileMetaData.length() || lastChunk) {
        try {
            Store.verify(indexOutput);
        } finally {
            // we are done
            indexOutput.close();
        }
        final String temporaryFileName = getTempNameForFile(name);
        assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName) : "expected: [" + temporaryFileName + "] in " + Arrays.toString(store.directory().listAll());
        store.directory().sync(Collections.singleton(temporaryFileName));
        IndexOutput remove = removeOpenIndexOutputs(name);
        // remove maybe null if we got finished
        assert remove == null || remove == indexOutput;
    }
}

Example 3 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class RecoveryStatusTests method testRenameTempFiles.

public void testRenameTempFiles() throws IOException {
    IndexService service = createIndex("foo");
    IndexShard indexShard = service.getShardOrNull(0);
    DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    RecoveryTarget status = new RecoveryTarget(indexShard, node, new PeerRecoveryTargetService.RecoveryListener() {

        @Override
        public void onRecoveryDone(RecoveryState state) {
        }

        @Override
        public void onRecoveryFailure(RecoveryState state, RecoveryFailedException e, boolean sendShardFailure) {
        }
    }, version -> {
    });
    try (IndexOutput indexOutput = status.openAndPutIndexOutput("foo.bar", new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw"), status.store())) {
        indexOutput.writeInt(1);
        IndexOutput openIndexOutput = status.getOpenIndexOutput("foo.bar");
        assertSame(openIndexOutput, indexOutput);
        openIndexOutput.writeInt(1);
        CodecUtil.writeFooter(indexOutput);
    }
    try {
        status.openAndPutIndexOutput("foo.bar", new StoreFileMetaData("foo.bar", 8 + CodecUtil.footerLength(), "9z51nw"), status.store());
        fail("file foo.bar is already opened and registered");
    } catch (IllegalStateException ex) {
        assertEquals("output for file [foo.bar] has already been created", ex.getMessage());
    // all well = it's already registered
    }
    status.removeOpenIndexOutputs("foo.bar");
    Set<String> strings = Sets.newHashSet(status.store().directory().listAll());
    String expectedFile = null;
    for (String file : strings) {
        if (Pattern.compile("recovery[.][\\w-]+[.]foo[.]bar").matcher(file).matches()) {
            expectedFile = file;
            break;
        }
    }
    assertNotNull(expectedFile);
    // we have to close it here otherwise rename fails since the write.lock is held by the engine
    indexShard.close("foo", false);
    status.renameAllTempFiles();
    strings = Sets.newHashSet(status.store().directory().listAll());
    assertTrue(strings.toString(), strings.contains("foo.bar"));
    assertFalse(strings.toString(), strings.contains(expectedFile));
    // we must fail the recovery because marking it as done will try to move the shard to POST_RECOVERY, which will fail because it's started
    status.fail(new RecoveryFailedException(status.state(), "end of test. OK.", null), false);
}

Example 4 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class StoreTests method testCheckIntegrity.

public void testCheckIntegrity() throws IOException {
    Directory dir = newDirectory();
    long luceneFileLength = 0;
    try (IndexOutput output = dir.createOutput("lucene_checksum.bin", IOContext.DEFAULT)) {
        int iters = scaledRandomIntBetween(10, 100);
        for (int i = 0; i < iters; i++) {
            BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
            output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
            luceneFileLength += bytesRef.length;
        }
        CodecUtil.writeFooter(output);
        luceneFileLength += CodecUtil.footerLength();
    }
    final long luceneChecksum;
    try (IndexInput indexInput = dir.openInput("lucene_checksum.bin", IOContext.DEFAULT)) {
        assertEquals(luceneFileLength, indexInput.length());
        luceneChecksum = CodecUtil.retrieveChecksum(indexInput);
    }
    dir.close();
}

Example 5 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project elasticsearch by elastic.

the class StoreTests method testVerifyingIndexInput.

public void testVerifyingIndexInput() throws IOException {
    Directory dir = newDirectory();
    IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
    int iters = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < iters; i++) {
        BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
        output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
    }
    CodecUtil.writeFooter(output);
    output.close();
    // Check file
    IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
    long checksum = CodecUtil.retrieveChecksum(indexInput);
    indexInput.seek(0);
    IndexInput verifyingIndexInput = new Store.VerifyingIndexInput(dir.openInput("foo.bar", IOContext.DEFAULT));
    readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
    Store.verify(verifyingIndexInput);
    assertThat(checksum, equalTo(((ChecksumIndexInput) verifyingIndexInput).getChecksum()));
    IOUtils.close(indexInput, verifyingIndexInput);
    // Corrupt file and check again
    corruptFile(dir, "foo.bar", "foo1.bar");
    verifyingIndexInput = new Store.VerifyingIndexInput(dir.openInput("foo1.bar", IOContext.DEFAULT));
    readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
    try {
        Store.verify(verifyingIndexInput);
        fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
    // ok
    }
    IOUtils.close(verifyingIndexInput);
    IOUtils.close(dir);
}