Examples with IndexOutputOutputStream org.elasticsearch.common.lucene.store.IndexOutputOutputStream used on opensource projects

Example 1 with IndexOutputOutputStream

use of org.elasticsearch.common.lucene.store.IndexOutputOutputStream in project elasticsearch by elastic.

the class ChecksumBlobStoreFormat method writeBlob.

/**
     * Writes blob in atomic manner without resolving the blobName using using {@link #blobName} method.
     * <p>
     * The blob will be compressed and checksum will be written if required.
     *
     * @param obj           object to be serialized
     * @param blobContainer blob container
     * @param blobName          blob name
     */
protected void writeBlob(T obj, BlobContainer blobContainer, String blobName) throws IOException {
    BytesReference bytes = write(obj);
    try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
        final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")";
        try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, blobName, byteArrayOutputStream, BUFFER_SIZE)) {
            CodecUtil.writeHeader(indexOutput, codec, VERSION);
            try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {

                @Override
                public void close() throws IOException {
                // this is important since some of the XContentBuilders write bytes on close.
                // in order to write the footer we need to prevent closing the actual index input.
                }
            }) {
                bytes.writeTo(indexOutputOutputStream);
            }
            CodecUtil.writeFooter(indexOutput);
        }
        BytesArray bytesArray = new BytesArray(byteArrayOutputStream.toByteArray());
        try (InputStream stream = bytesArray.streamInput()) {
            blobContainer.writeBlob(blobName, stream, bytesArray.length());
        }
    }
}

Example 2 with IndexOutputOutputStream

use of org.elasticsearch.common.lucene.store.IndexOutputOutputStream in project crate by crate.

the class MetadataStateFormat method writeStateToFirstLocation.

private void writeStateToFirstLocation(final T state, Path stateLocation, Directory stateDir, String tmpFileName) throws WriteStateException {
    try {
        deleteFileIfExists(stateLocation, stateDir, tmpFileName);
        try (IndexOutput out = stateDir.createOutput(tmpFileName, IOContext.DEFAULT)) {
            CodecUtil.writeHeader(out, STATE_FILE_CODEC, STATE_FILE_VERSION);
            out.writeInt(FORMAT.index());
            try (XContentBuilder builder = newXContentBuilder(FORMAT, new IndexOutputOutputStream(out) {

                @Override
                public void close() {
                // this is important since some of the XContentBuilders write bytes on close.
                // in order to write the footer we need to prevent closing the actual index input.
                }
            })) {
                builder.startObject();
                toXContent(builder, state);
                builder.endObject();
            }
            CodecUtil.writeFooter(out);
        }
        stateDir.sync(Collections.singleton(tmpFileName));
    } catch (Exception e) {
        throw new WriteStateException(false, "failed to write state to the first location tmp file " + stateLocation.resolve(tmpFileName), e);
    }
}

Example 3 with IndexOutputOutputStream

use of org.elasticsearch.common.lucene.store.IndexOutputOutputStream in project crate by crate.

the class RecoverySourceHandlerTests method generateFiles.

private List<StoreFileMetadata> generateFiles(Store store, int numFiles, IntSupplier fileSizeSupplier) throws IOException {
    List<StoreFileMetadata> files = new ArrayList<>();
    for (int i = 0; i < numFiles; i++) {
        byte[] buffer = randomByteArrayOfLength(fileSizeSupplier.getAsInt());
        CRC32 digest = new CRC32();
        digest.update(buffer, 0, buffer.length);
        StoreFileMetadata md = new StoreFileMetadata("test-" + i, buffer.length + 8, Store.digestToString(digest.getValue()), org.apache.lucene.util.Version.LATEST);
        try (OutputStream out = new IndexOutputOutputStream(store.createVerifyingOutput(md.name(), md, IOContext.DEFAULT))) {
            out.write(buffer);
            out.write(Numbers.longToBytes(digest.getValue()));
        }
        store.directory().sync(Collections.singleton(md.name()));
        files.add(md);
    }
    return files;
}

Example 4 with IndexOutputOutputStream

use of org.elasticsearch.common.lucene.store.IndexOutputOutputStream in project elasticsearch by elastic.

the class RecoverySourceHandlerTests method testHandleCorruptedIndexOnSendSendFiles.

public void testHandleCorruptedIndexOnSendSendFiles() throws Throwable {
    Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).put("indices.recovery.concurrent_small_file_streams", 1).build();
    final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
    final StartRecoveryRequest request = new StartRecoveryRequest(shardId, new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), null, randomBoolean(), randomNonNegativeLong(), randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : 0L);
    Path tempDir = createTempDir();
    Store store = newStore(tempDir, false);
    AtomicBoolean failedEngine = new AtomicBoolean(false);
    RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {
    }, recoverySettings.getChunkSize().bytesAsInt(), Settings.EMPTY) {

        @Override
        protected void failEngine(IOException cause) {
            assertFalse(failedEngine.get());
            failedEngine.set(true);
        }
    };
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata(null);
    List<StoreFileMetaData> metas = new ArrayList<>();
    for (StoreFileMetaData md : metadata) {
        metas.add(md);
    }
    CorruptionUtils.corruptFile(random(), FileSystemUtils.files(tempDir, (p) -> (p.getFileName().toString().equals("write.lock") || p.getFileName().toString().startsWith("extra")) == false));
    Store targetStore = newStore(createTempDir(), false);
    try {
        handler.sendFiles(store, metas.toArray(new StoreFileMetaData[0]), (md) -> {
            try {
                return new IndexOutputOutputStream(targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) {

                    @Override
                    public void close() throws IOException {
                        super.close();
                        // sync otherwise MDW will mess with it
                        store.directory().sync(Collections.singleton(md.name()));
                    }
                };
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        });
        fail("corrupted index");
    } catch (IOException ex) {
        assertNotNull(ExceptionsHelper.unwrapCorruption(ex));
    }
    assertTrue(failedEngine.get());
    IOUtils.close(store, targetStore);
}

Example 5 with IndexOutputOutputStream

use of org.elasticsearch.common.lucene.store.IndexOutputOutputStream in project elasticsearch by elastic.

the class RecoverySourceHandlerTests method testSendFiles.

public void testSendFiles() throws Throwable {
    Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).put("indices.recovery.concurrent_small_file_streams", 1).build();
    final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
    final StartRecoveryRequest request = new StartRecoveryRequest(shardId, new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), null, randomBoolean(), randomNonNegativeLong(), randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : randomNonNegativeLong());
    Store store = newStore(createTempDir());
    RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {
    }, recoverySettings.getChunkSize().bytesAsInt(), Settings.EMPTY);
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata(null);
    List<StoreFileMetaData> metas = new ArrayList<>();
    for (StoreFileMetaData md : metadata) {
        metas.add(md);
    }
    Store targetStore = newStore(createTempDir());
    handler.sendFiles(store, metas.toArray(new StoreFileMetaData[0]), (md) -> {
        try {
            return new IndexOutputOutputStream(targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) {

                @Override
                public void close() throws IOException {
                    super.close();
                    // sync otherwise MDW will mess with it
                    targetStore.directory().sync(Collections.singleton(md.name()));
                }
            };
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    });
    Store.MetadataSnapshot targetStoreMetadata = targetStore.getMetadata(null);
    Store.RecoveryDiff recoveryDiff = targetStoreMetadata.recoveryDiff(metadata);
    assertEquals(metas.size(), recoveryDiff.identical.size());
    assertEquals(0, recoveryDiff.different.size());
    assertEquals(0, recoveryDiff.missing.size());
    IndexReader reader = DirectoryReader.open(targetStore.directory());
    assertEquals(numDocs, reader.maxDoc());
    IOUtils.close(reader, store, targetStore);
}