Search in sources :

Example 41 with StoreFileMetaData

use of org.elasticsearch.index.store.StoreFileMetaData in project elasticsearch by elastic.

the class FileInfoTests method testToFromXContent.

public void testToFromXContent() throws IOException {
    final int iters = scaledRandomIntBetween(1, 10);
    for (int iter = 0; iter < iters; iter++) {
        final BytesRef hash = new BytesRef(scaledRandomIntBetween(0, 1024 * 1024));
        hash.length = hash.bytes.length;
        for (int i = 0; i < hash.length; i++) {
            hash.bytes[i] = randomByte();
        }
        StoreFileMetaData meta = new StoreFileMetaData("foobar", Math.abs(randomLong()), randomAsciiOfLengthBetween(1, 10), Version.LATEST, hash);
        ByteSizeValue size = new ByteSizeValue(Math.abs(randomLong()));
        BlobStoreIndexShardSnapshot.FileInfo info = new BlobStoreIndexShardSnapshot.FileInfo("_foobar", meta, size);
        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
        BlobStoreIndexShardSnapshot.FileInfo.toXContent(info, builder, ToXContent.EMPTY_PARAMS);
        byte[] xcontent = BytesReference.toBytes(shuffleXContent(builder).bytes());
        final BlobStoreIndexShardSnapshot.FileInfo parsedInfo;
        try (XContentParser parser = createParser(JsonXContent.jsonXContent, xcontent)) {
            parser.nextToken();
            parsedInfo = BlobStoreIndexShardSnapshot.FileInfo.fromXContent(parser);
        }
        assertThat(info.name(), equalTo(parsedInfo.name()));
        assertThat(info.physicalName(), equalTo(parsedInfo.physicalName()));
        assertThat(info.length(), equalTo(parsedInfo.length()));
        assertThat(info.checksum(), equalTo(parsedInfo.checksum()));
        assertThat(info.partSize(), equalTo(parsedInfo.partSize()));
        assertThat(parsedInfo.metadata().hash().length, equalTo(hash.length));
        assertThat(parsedInfo.metadata().hash(), equalTo(hash));
        assertThat(parsedInfo.metadata().writtenBy(), equalTo(Version.LATEST));
        assertThat(parsedInfo.isSame(info.metadata()), is(true));
    }
}
Also used : FileInfo(org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo) StoreFileMetaData(org.elasticsearch.index.store.StoreFileMetaData) ByteSizeValue(org.elasticsearch.common.unit.ByteSizeValue) FileInfo(org.elasticsearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot.FileInfo) BytesRef(org.apache.lucene.util.BytesRef) XContentBuilder(org.elasticsearch.common.xcontent.XContentBuilder) XContentParser(org.elasticsearch.common.xcontent.XContentParser)

Example 42 with StoreFileMetaData

use of org.elasticsearch.index.store.StoreFileMetaData in project elasticsearch by elastic.

the class RecoverySourceHandlerTests method testHandleCorruptedIndexOnSendSendFiles.

public void testHandleCorruptedIndexOnSendSendFiles() throws Throwable {
    Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).put("indices.recovery.concurrent_small_file_streams", 1).build();
    final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
    final StartRecoveryRequest request = new StartRecoveryRequest(shardId, new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), null, randomBoolean(), randomNonNegativeLong(), randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : 0L);
    Path tempDir = createTempDir();
    Store store = newStore(tempDir, false);
    AtomicBoolean failedEngine = new AtomicBoolean(false);
    RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {
    }, recoverySettings.getChunkSize().bytesAsInt(), Settings.EMPTY) {

        @Override
        protected void failEngine(IOException cause) {
            assertFalse(failedEngine.get());
            failedEngine.set(true);
        }
    };
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata(null);
    List<StoreFileMetaData> metas = new ArrayList<>();
    for (StoreFileMetaData md : metadata) {
        metas.add(md);
    }
    CorruptionUtils.corruptFile(random(), FileSystemUtils.files(tempDir, (p) -> (p.getFileName().toString().equals("write.lock") || p.getFileName().toString().startsWith("extra")) == false));
    Store targetStore = newStore(createTempDir(), false);
    try {
        handler.sendFiles(store, metas.toArray(new StoreFileMetaData[0]), (md) -> {
            try {
                return new IndexOutputOutputStream(targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) {

                    @Override
                    public void close() throws IOException {
                        super.close();
                        // sync otherwise MDW will mess with it
                        store.directory().sync(Collections.singleton(md.name()));
                    }
                };
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        });
        fail("corrupted index");
    } catch (IOException ex) {
        assertNotNull(ExceptionsHelper.unwrapCorruption(ex));
    }
    assertTrue(failedEngine.get());
    IOUtils.close(store, targetStore);
}
Also used : Path(java.nio.file.Path) ShardId(org.elasticsearch.index.shard.ShardId) Versions(org.elasticsearch.common.lucene.uid.Versions) Arrays(java.util.Arrays) IndexSettingsModule(org.elasticsearch.test.IndexSettingsModule) Term(org.apache.lucene.index.Term) ParseContext(org.elasticsearch.index.mapper.ParseContext) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) SeqNoStats(org.elasticsearch.index.seqno.SeqNoStats) Document(org.apache.lucene.document.Document) Matchers.anyBoolean(org.mockito.Matchers.anyBoolean) Settings(org.elasticsearch.common.settings.Settings) Directory(org.apache.lucene.store.Directory) Mockito.doAnswer(org.mockito.Mockito.doAnswer) UidFieldMapper(org.elasticsearch.index.mapper.UidFieldMapper) IOContext(org.apache.lucene.store.IOContext) Path(java.nio.file.Path) Releasable(org.elasticsearch.common.lease.Releasable) NumericDocValuesField(org.apache.lucene.document.NumericDocValuesField) DirectoryReader(org.apache.lucene.index.DirectoryReader) BytesReference(org.elasticsearch.common.bytes.BytesReference) Engine(org.elasticsearch.index.engine.Engine) Matchers.any(org.mockito.Matchers.any) SeqNoFieldMapper(org.elasticsearch.index.mapper.SeqNoFieldMapper) List(java.util.List) Version(org.elasticsearch.Version) IndexMetaData(org.elasticsearch.cluster.metadata.IndexMetaData) Matchers.equalTo(org.hamcrest.Matchers.equalTo) IndexOutputOutputStream(org.elasticsearch.common.lucene.store.IndexOutputOutputStream) IndexReader(org.apache.lucene.index.IndexReader) Mockito.mock(org.mockito.Mockito.mock) IndexCommit(org.apache.lucene.index.IndexCommit) XContentType(org.elasticsearch.common.xcontent.XContentType) StringField(org.apache.lucene.document.StringField) IndexShardRelocatedException(org.elasticsearch.index.shard.IndexShardRelocatedException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) SegmentsStats(org.elasticsearch.index.engine.SegmentsStats) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) BytesArray(org.elasticsearch.common.bytes.BytesArray) BaseDirectoryWrapper(org.apache.lucene.store.BaseDirectoryWrapper) StoreFileMetaData(org.elasticsearch.index.store.StoreFileMetaData) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) SequenceNumbersService(org.elasticsearch.index.seqno.SequenceNumbersService) Store(org.elasticsearch.index.store.Store) IndexSettings(org.elasticsearch.index.IndexSettings) Mapping(org.elasticsearch.index.mapper.Mapping) ESTestCase(org.elasticsearch.test.ESTestCase) Collections.emptyMap(java.util.Collections.emptyMap) FileSystemUtils(org.elasticsearch.common.io.FileSystemUtils) IndexShardState(org.elasticsearch.index.shard.IndexShardState) CorruptionUtils(org.elasticsearch.test.CorruptionUtils) Uid(org.elasticsearch.index.mapper.Uid) Collections.emptySet(java.util.Collections.emptySet) IndexShard(org.elasticsearch.index.shard.IndexShard) IOUtils(org.apache.lucene.util.IOUtils) IOException(java.io.IOException) ParsedDocument(org.elasticsearch.index.mapper.ParsedDocument) Mockito.when(org.mockito.Mockito.when) DirectoryService(org.elasticsearch.index.store.DirectoryService) ExceptionsHelper(org.elasticsearch.ExceptionsHelper) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) Field(org.apache.lucene.document.Field) Translog(org.elasticsearch.index.translog.Translog) TextField(org.apache.lucene.document.TextField) DummyShardLock(org.elasticsearch.test.DummyShardLock) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) Collections(java.util.Collections) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) IndexOutputOutputStream(org.elasticsearch.common.lucene.store.IndexOutputOutputStream) ArrayList(java.util.ArrayList) Store(org.elasticsearch.index.store.Store) IOException(java.io.IOException) Document(org.apache.lucene.document.Document) ParsedDocument(org.elasticsearch.index.mapper.ParsedDocument) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) StoreFileMetaData(org.elasticsearch.index.store.StoreFileMetaData) StringField(org.apache.lucene.document.StringField) Settings(org.elasticsearch.common.settings.Settings) IndexSettings(org.elasticsearch.index.IndexSettings) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) Directory(org.apache.lucene.store.Directory)

Example 43 with StoreFileMetaData

use of org.elasticsearch.index.store.StoreFileMetaData in project elasticsearch by elastic.

the class RecoverySourceHandlerTests method testSendFiles.

public void testSendFiles() throws Throwable {
    Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).put("indices.recovery.concurrent_small_file_streams", 1).build();
    final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
    final StartRecoveryRequest request = new StartRecoveryRequest(shardId, new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), null, randomBoolean(), randomNonNegativeLong(), randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : randomNonNegativeLong());
    Store store = newStore(createTempDir());
    RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {
    }, recoverySettings.getChunkSize().bytesAsInt(), Settings.EMPTY);
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata(null);
    List<StoreFileMetaData> metas = new ArrayList<>();
    for (StoreFileMetaData md : metadata) {
        metas.add(md);
    }
    Store targetStore = newStore(createTempDir());
    handler.sendFiles(store, metas.toArray(new StoreFileMetaData[0]), (md) -> {
        try {
            return new IndexOutputOutputStream(targetStore.createVerifyingOutput(md.name(), md, IOContext.DEFAULT)) {

                @Override
                public void close() throws IOException {
                    super.close();
                    // sync otherwise MDW will mess with it
                    targetStore.directory().sync(Collections.singleton(md.name()));
                }
            };
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    });
    Store.MetadataSnapshot targetStoreMetadata = targetStore.getMetadata(null);
    Store.RecoveryDiff recoveryDiff = targetStoreMetadata.recoveryDiff(metadata);
    assertEquals(metas.size(), recoveryDiff.identical.size());
    assertEquals(0, recoveryDiff.different.size());
    assertEquals(0, recoveryDiff.missing.size());
    IndexReader reader = DirectoryReader.open(targetStore.directory());
    assertEquals(numDocs, reader.maxDoc());
    IOUtils.close(reader, store, targetStore);
}
Also used : DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) IndexOutputOutputStream(org.elasticsearch.common.lucene.store.IndexOutputOutputStream) ArrayList(java.util.ArrayList) Store(org.elasticsearch.index.store.Store) IOException(java.io.IOException) Document(org.apache.lucene.document.Document) ParsedDocument(org.elasticsearch.index.mapper.ParsedDocument) StoreFileMetaData(org.elasticsearch.index.store.StoreFileMetaData) StringField(org.apache.lucene.document.StringField) IndexReader(org.apache.lucene.index.IndexReader) Settings(org.elasticsearch.common.settings.Settings) IndexSettings(org.elasticsearch.index.IndexSettings) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) Directory(org.apache.lucene.store.Directory)

Example 44 with StoreFileMetaData

use of org.elasticsearch.index.store.StoreFileMetaData in project elasticsearch by elastic.

the class ReplicaShardAllocator method computeMatchingBytes.

private static long computeMatchingBytes(TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore, TransportNodesListShardStoreMetaData.StoreFilesMetaData storeFilesMetaData) {
    String primarySyncId = primaryStore.syncId();
    String replicaSyncId = storeFilesMetaData.syncId();
    // see if we have a sync id we can make use of
    if (replicaSyncId != null && replicaSyncId.equals(primarySyncId)) {
        return Long.MAX_VALUE;
    } else {
        long sizeMatched = 0;
        for (StoreFileMetaData storeFileMetaData : storeFilesMetaData) {
            String metaDataFileName = storeFileMetaData.name();
            if (primaryStore.fileExists(metaDataFileName) && primaryStore.file(metaDataFileName).isSame(storeFileMetaData)) {
                sizeMatched += storeFileMetaData.length();
            }
        }
        return sizeMatched;
    }
}
Also used : StoreFileMetaData(org.elasticsearch.index.store.StoreFileMetaData)

Example 45 with StoreFileMetaData

use of org.elasticsearch.index.store.StoreFileMetaData in project crate by crate.

the class RecoverySourceHandler method sendFiles.

void sendFiles(Store store, StoreFileMetadata[] files, IntSupplier translogOps, ActionListener<Void> listener) {
    // send smallest first
    ArrayUtil.timSort(files, Comparator.comparingLong(StoreFileMetadata::length));
    final MultiChunkTransfer<StoreFileMetadata, FileChunk> multiFileSender = new MultiChunkTransfer<StoreFileMetadata, FileChunk>(logger, listener, maxConcurrentFileChunks, Arrays.asList(files)) {

        final Deque<byte[]> buffers = new ConcurrentLinkedDeque<>();

        InputStreamIndexInput currentInput = null;

        long offset = 0;

        @Override
        protected void onNewResource(StoreFileMetadata md) throws IOException {
            offset = 0;
            IOUtils.close(currentInput, () -> currentInput = null);
            final IndexInput indexInput = store.directory().openInput(md.name(), IOContext.READONCE);
            currentInput = new InputStreamIndexInput(indexInput, md.length()) {

                @Override
                public void close() throws IOException {
                    // InputStreamIndexInput's close is a noop
                    IOUtils.close(indexInput, super::close);
                }
            };
        }

        private byte[] acquireBuffer() {
            final byte[] buffer = buffers.pollFirst();
            if (buffer != null) {
                return buffer;
            }
            return new byte[chunkSizeInBytes];
        }

        @Override
        protected FileChunk nextChunkRequest(StoreFileMetadata md) throws IOException {
            assert Transports.assertNotTransportThread("read file chunk");
            cancellableThreads.checkForCancel();
            final byte[] buffer = acquireBuffer();
            final int bytesRead = currentInput.read(buffer);
            if (bytesRead == -1) {
                throw new CorruptIndexException("file truncated; length=" + md.length() + " offset=" + offset, md.name());
            }
            final boolean lastChunk = offset + bytesRead == md.length();
            final FileChunk chunk = new FileChunk(md, new BytesArray(buffer, 0, bytesRead), offset, lastChunk, () -> buffers.addFirst(buffer));
            offset += bytesRead;
            return chunk;
        }

        @Override
        protected void executeChunkRequest(FileChunk request, ActionListener<Void> listener) {
            cancellableThreads.checkForCancel();
            recoveryTarget.writeFileChunk(request.md, request.position, request.content, request.lastChunk, translogOps.getAsInt(), ActionListener.runBefore(listener, request::close));
        }

        @Override
        protected void handleError(StoreFileMetadata md, Exception e) throws Exception {
            handleErrorOnSendFiles(store, e, new StoreFileMetadata[] { md });
        }

        @Override
        public void close() throws IOException {
            IOUtils.close(currentInput, () -> currentInput = null);
        }
    };
    resources.add(multiFileSender);
    multiFileSender.start();
}
Also used : BytesArray(org.elasticsearch.common.bytes.BytesArray) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) StoreFileMetadata(org.elasticsearch.index.store.StoreFileMetadata) IOException(java.io.IOException) Deque(java.util.Deque) ConcurrentLinkedDeque(java.util.concurrent.ConcurrentLinkedDeque) IndexFormatTooNewException(org.apache.lucene.index.IndexFormatTooNewException) RecoveryEngineException(org.elasticsearch.index.engine.RecoveryEngineException) RetentionLeaseNotFoundException(org.elasticsearch.index.seqno.RetentionLeaseNotFoundException) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) RemoteTransportException(org.elasticsearch.transport.RemoteTransportException) IndexShardClosedException(org.elasticsearch.index.shard.IndexShardClosedException) IndexShardRelocatedException(org.elasticsearch.index.shard.IndexShardRelocatedException) IOException(java.io.IOException) IndexFormatTooOldException(org.apache.lucene.index.IndexFormatTooOldException) ThreadedActionListener(org.elasticsearch.action.support.ThreadedActionListener) ActionListener(org.elasticsearch.action.ActionListener) InputStreamIndexInput(org.elasticsearch.common.lucene.store.InputStreamIndexInput) InputStreamIndexInput(org.elasticsearch.common.lucene.store.InputStreamIndexInput) IndexInput(org.apache.lucene.store.IndexInput)

Aggregations

StoreFileMetadata (org.elasticsearch.index.store.StoreFileMetadata)34 RoutingAllocation (org.elasticsearch.cluster.routing.allocation.RoutingAllocation)30 StoreFileMetaData (org.elasticsearch.index.store.StoreFileMetaData)25 IOException (java.io.IOException)19 ArrayList (java.util.ArrayList)18 Store (org.elasticsearch.index.store.Store)17 DiscoveryNode (org.elasticsearch.cluster.node.DiscoveryNode)16 CorruptIndexException (org.apache.lucene.index.CorruptIndexException)13 Directory (org.apache.lucene.store.Directory)10 RetentionLease (org.elasticsearch.index.seqno.RetentionLease)10 IndexShardRelocatedException (org.elasticsearch.index.shard.IndexShardRelocatedException)10 List (java.util.List)9 CopyOnWriteArrayList (java.util.concurrent.CopyOnWriteArrayList)9 Document (org.apache.lucene.document.Document)9 StringField (org.apache.lucene.document.StringField)9 RandomIndexWriter (org.apache.lucene.index.RandomIndexWriter)9 BytesArray (org.elasticsearch.common.bytes.BytesArray)9 RecoveryEngineException (org.elasticsearch.index.engine.RecoveryEngineException)9 ParsedDocument (org.elasticsearch.index.mapper.ParsedDocument)9 Settings (org.elasticsearch.common.settings.Settings)8