Examples with StoreFileMetadata org.elasticsearch.index.store.StoreFileMetadata used on opensource projects

Example 46 with StoreFileMetadata

use of org.elasticsearch.index.store.StoreFileMetadata in project crate by crate.

the class RecoverySourceHandler method phase1.

/**
 * Perform phase1 of the recovery operations. Once this {@link IndexCommit}
 * snapshot has been performed no commit operations (files being fsync'd)
 * are effectively allowed on this index until all recovery phases are done
 * <p>
 * Phase1 examines the segment files on the target node and copies over the
 * segments that are missing. Only segments that have the same size and
 * checksum can be reused
 */
void phase1(IndexCommit snapshot, long startingSeqNo, IntSupplier translogOps, ActionListener<SendFileResult> listener) {
    cancellableThreads.checkForCancel();
    final Store store = shard.store();
    try {
        final StopWatch stopWatch = new StopWatch().start();
        final Store.MetadataSnapshot recoverySourceMetadata;
        try {
            recoverySourceMetadata = store.getMetadata(snapshot);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            shard.failShard("recovery", ex);
            throw ex;
        }
        for (String name : snapshot.getFileNames()) {
            final StoreFileMetadata md = recoverySourceMetadata.get(name);
            if (md == null) {
                logger.info("Snapshot differs from actual index for file: {} meta: {}", name, recoverySourceMetadata.asMap());
                throw new CorruptIndexException("Snapshot differs from actual index - maybe index was removed metadata has " + recoverySourceMetadata.asMap().size() + " files", name);
            }
        }
        if (canSkipPhase1(recoverySourceMetadata, request.metadataSnapshot()) == false) {
            final List<String> phase1FileNames = new ArrayList<>();
            final List<Long> phase1FileSizes = new ArrayList<>();
            final List<String> phase1ExistingFileNames = new ArrayList<>();
            final List<Long> phase1ExistingFileSizes = new ArrayList<>();
            // Total size of segment files that are recovered
            long totalSizeInBytes = 0;
            // Total size of segment files that were able to be re-used
            long existingTotalSizeInBytes = 0;
            // Generate a "diff" of all the identical, different, and missing
            // segment files on the target node, using the existing files on
            // the source node
            final Store.RecoveryDiff diff = recoverySourceMetadata.recoveryDiff(request.metadataSnapshot());
            for (StoreFileMetadata md : diff.identical) {
                phase1ExistingFileNames.add(md.name());
                phase1ExistingFileSizes.add(md.length());
                existingTotalSizeInBytes += md.length();
                if (logger.isTraceEnabled()) {
                    logger.trace("recovery [phase1]: not recovering [{}], exist in local store and has checksum [{}]," + " size [{}]", md.name(), md.checksum(), md.length());
                }
                totalSizeInBytes += md.length();
            }
            List<StoreFileMetadata> phase1Files = new ArrayList<>(diff.different.size() + diff.missing.size());
            phase1Files.addAll(diff.different);
            phase1Files.addAll(diff.missing);
            for (StoreFileMetadata md : phase1Files) {
                if (request.metadataSnapshot().asMap().containsKey(md.name())) {
                    logger.trace("recovery [phase1]: recovering [{}], exists in local store, but is different: remote [{}], local [{}]", md.name(), request.metadataSnapshot().asMap().get(md.name()), md);
                } else {
                    logger.trace("recovery [phase1]: recovering [{}], does not exist in remote", md.name());
                }
                phase1FileNames.add(md.name());
                phase1FileSizes.add(md.length());
                totalSizeInBytes += md.length();
            }
            logger.trace("recovery [phase1]: recovering_files [{}] with total_size [{}], reusing_files [{}] with total_size [{}]", phase1FileNames.size(), new ByteSizeValue(totalSizeInBytes), phase1ExistingFileNames.size(), new ByteSizeValue(existingTotalSizeInBytes));
            final StepListener<Void> sendFileInfoStep = new StepListener<>();
            final StepListener<Void> sendFilesStep = new StepListener<>();
            final StepListener<RetentionLease> createRetentionLeaseStep = new StepListener<>();
            final StepListener<Void> cleanFilesStep = new StepListener<>();
            cancellableThreads.checkForCancel();
            recoveryTarget.receiveFileInfo(phase1FileNames, phase1FileSizes, phase1ExistingFileNames, phase1ExistingFileSizes, translogOps.getAsInt(), sendFileInfoStep);
            sendFileInfoStep.whenComplete(r -> sendFiles(store, phase1Files.toArray(new StoreFileMetadata[0]), translogOps, sendFilesStep), listener::onFailure);
            sendFilesStep.whenComplete(r -> createRetentionLease(startingSeqNo, createRetentionLeaseStep), listener::onFailure);
            createRetentionLeaseStep.whenComplete(retentionLease -> {
                final long lastKnownGlobalCheckpoint = shard.getLastKnownGlobalCheckpoint();
                assert retentionLease == null || retentionLease.retainingSequenceNumber() - 1 <= lastKnownGlobalCheckpoint : retentionLease + " vs " + lastKnownGlobalCheckpoint;
                // Establishes new empty translog on the replica with global checkpoint set to lastKnownGlobalCheckpoint. We want
                // the commit we just copied to be a safe commit on the replica, so why not set the global checkpoint on the replica
                // to the max seqno of this commit? Because (in rare corner cases) this commit might not be a safe commit here on
                // the primary, and in these cases the max seqno would be too high to be valid as a global checkpoint.
                cleanFiles(store, recoverySourceMetadata, translogOps, lastKnownGlobalCheckpoint, cleanFilesStep);
            }, listener::onFailure);
            final long totalSize = totalSizeInBytes;
            final long existingTotalSize = existingTotalSizeInBytes;
            cleanFilesStep.whenComplete(r -> {
                final TimeValue took = stopWatch.totalTime();
                logger.trace("recovery [phase1]: took [{}]", took);
                listener.onResponse(new SendFileResult(phase1FileNames, phase1FileSizes, totalSize, phase1ExistingFileNames, phase1ExistingFileSizes, existingTotalSize, took));
            }, listener::onFailure);
        } else {
            logger.trace("skipping [phase1] since source and target have identical sync id [{}]", recoverySourceMetadata.getSyncId());
            // but we must still create a retention lease
            final StepListener<RetentionLease> createRetentionLeaseStep = new StepListener<>();
            createRetentionLease(startingSeqNo, createRetentionLeaseStep);
            createRetentionLeaseStep.whenComplete(retentionLease -> {
                final TimeValue took = stopWatch.totalTime();
                logger.trace("recovery [phase1]: took [{}]", took);
                listener.onResponse(new SendFileResult(Collections.emptyList(), Collections.emptyList(), 0L, Collections.emptyList(), Collections.emptyList(), 0L, took));
            }, listener::onFailure);
        }
    } catch (Exception e) {
        throw new RecoverFilesRecoveryException(request.shardId(), 0, new ByteSizeValue(0L), e);
    }
}

Example 47 with StoreFileMetadata

use of org.elasticsearch.index.store.StoreFileMetadata in project crate by crate.

the class RecoverySourceHandler method handleErrorOnSendFiles.

private void handleErrorOnSendFiles(Store store, Exception e, StoreFileMetadata[] mds) throws Exception {
    final IOException corruptIndexException = ExceptionsHelper.unwrapCorruption(e);
    assert Transports.assertNotTransportThread(RecoverySourceHandler.this + "[handle error on send/clean files]");
    if (corruptIndexException != null) {
        Exception localException = null;
        for (StoreFileMetadata md : mds) {
            cancellableThreads.checkForCancel();
            logger.debug("checking integrity for file {} after remove corruption exception", md);
            if (store.checkIntegrityNoException(md) == false) {
                // we are corrupted on the primary -- fail!
                logger.warn("{} Corrupted file detected {} checksum mismatch", shardId, md);
                if (localException == null) {
                    localException = corruptIndexException;
                }
                failEngine(corruptIndexException);
            }
        }
        if (localException != null) {
            throw localException;
        } else {
            // corruption has happened on the way to replica
            RemoteTransportException remoteException = new RemoteTransportException("File corruption occurred on recovery but checksums are ok", null);
            remoteException.addSuppressed(e);
            logger.warn(() -> new ParameterizedMessage("{} Remote file corruption on node {}, recovering {}. local checksum OK", shardId, request.targetNode(), mds), corruptIndexException);
            throw remoteException;
        }
    }
    throw e;
}

Example 48 with StoreFileMetadata

use of org.elasticsearch.index.store.StoreFileMetadata in project crate by crate.

the class IndexShard method doCheckIndex.

private void doCheckIndex() throws IOException {
    final long timeNS = System.nanoTime();
    if (!Lucene.indexExists(store.directory())) {
        return;
    }
    BytesStreamOutput os = new BytesStreamOutput();
    PrintStream out = new PrintStream(os, false, StandardCharsets.UTF_8.name());
    if ("checksum".equals(checkIndexOnStartup)) {
        // physical verification only: verify all checksums for the latest commit
        IOException corrupt = null;
        MetadataSnapshot metadata = snapshotStoreMetadata();
        for (Map.Entry<String, StoreFileMetadata> entry : metadata.asMap().entrySet()) {
            try {
                Store.checkIntegrity(entry.getValue(), store.directory());
                out.println("checksum passed: " + entry.getKey());
            } catch (IOException exc) {
                out.println("checksum failed: " + entry.getKey());
                exc.printStackTrace(out);
                corrupt = exc;
            }
        }
        out.flush();
        if (corrupt != null) {
            logger.warn("check index [failure]\n{}", os.bytes().utf8ToString());
            throw corrupt;
        }
    } else {
        // full checkindex
        final CheckIndex.Status status = store.checkIndex(out);
        out.flush();
        if (!status.clean) {
            if (state == IndexShardState.CLOSED) {
                // ignore if closed....
                return;
            }
            logger.warn("check index [failure]\n{}", os.bytes().utf8ToString());
            throw new IOException("index check failure");
        }
    }
    if (logger.isDebugEnabled()) {
        logger.debug("check index [success]\n{}", os.bytes().utf8ToString());
    }
    recoveryState.getVerifyIndex().checkIndexTime(Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - timeNS)));
}

Example 49 with StoreFileMetadata

use of org.elasticsearch.index.store.StoreFileMetadata in project crate by crate.

the class RecoverySourceHandlerTests method testSendFileChunksStopOnError.

@Test
public void testSendFileChunksStopOnError() throws Exception {
    final List<FileChunkResponse> unrepliedChunks = new CopyOnWriteArrayList<>();
    final AtomicInteger sentChunks = new AtomicInteger();
    final TestRecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {

        final AtomicLong chunkNumberGenerator = new AtomicLong();

        @Override
        public void writeFileChunk(StoreFileMetadata md, long position, BytesReference content, boolean lastChunk, int totalTranslogOps, ActionListener<Void> listener) {
            final long chunkNumber = chunkNumberGenerator.getAndIncrement();
            logger.info("--> write chunk name={} seq={}, position={}", md.name(), chunkNumber, position);
            unrepliedChunks.add(new FileChunkResponse(chunkNumber, listener));
            sentChunks.incrementAndGet();
        }
    };
    final int maxConcurrentChunks = between(1, 4);
    final int chunkSize = between(1, 16);
    final RecoverySourceHandler handler = new RecoverySourceHandler(null, new AsyncRecoveryTarget(recoveryTarget, recoveryExecutor), threadPool, getStartRecoveryRequest(), chunkSize, maxConcurrentChunks, between(1, 5));
    Store store = newStore(createTempDir(), false);
    List<StoreFileMetadata> files = generateFiles(store, between(1, 10), () -> between(1, chunkSize * 20));
    int totalChunks = files.stream().mapToInt(md -> ((int) md.length() + chunkSize - 1) / chunkSize).sum();
    SetOnce<Exception> sendFilesError = new SetOnce<>();
    CountDownLatch sendFilesLatch = new CountDownLatch(1);
    handler.sendFiles(store, files.toArray(new StoreFileMetadata[0]), () -> 0, new LatchedActionListener<>(ActionListener.wrap(r -> sendFilesError.set(null), e -> sendFilesError.set(e)), sendFilesLatch));
    assertBusy(() -> assertThat(sentChunks.get(), equalTo(Math.min(totalChunks, maxConcurrentChunks))));
    List<FileChunkResponse> failedChunks = randomSubsetOf(between(1, unrepliedChunks.size()), unrepliedChunks);
    CountDownLatch replyLatch = new CountDownLatch(failedChunks.size());
    failedChunks.forEach(c -> {
        c.listener.onFailure(new IllegalStateException("test chunk exception"));
        replyLatch.countDown();
    });
    replyLatch.await();
    unrepliedChunks.removeAll(failedChunks);
    unrepliedChunks.forEach(c -> {
        if (randomBoolean()) {
            c.listener.onFailure(new RuntimeException("test"));
        } else {
            c.listener.onResponse(null);
        }
    });
    sendFilesLatch.await();
    assertThat(sendFilesError.get(), instanceOf(IllegalStateException.class));
    assertThat(sendFilesError.get().getMessage(), containsString("test chunk exception"));
    assertThat("no more chunks should be sent", sentChunks.get(), equalTo(Math.min(totalChunks, maxConcurrentChunks)));
    store.close();
}

Example 50 with StoreFileMetadata

use of org.elasticsearch.index.store.StoreFileMetadata in project crate by crate.

the class RecoverySourceHandlerTests method testHandleCorruptedIndexOnSendSendFiles.

@Test
public void testHandleCorruptedIndexOnSendSendFiles() throws Throwable {
    Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).put("indices.recovery.concurrent_small_file_streams", 1).build();
    final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
    final StartRecoveryRequest request = getStartRecoveryRequest();
    Path tempDir = createTempDir();
    Store store = newStore(tempDir, false);
    AtomicBoolean failedEngine = new AtomicBoolean(false);
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata(null);
    List<StoreFileMetadata> metas = new ArrayList<>();
    for (StoreFileMetadata md : metadata) {
        metas.add(md);
    }
    CorruptionUtils.corruptFile(random(), FileSystemUtils.files(tempDir, (p) -> (p.getFileName().toString().equals("write.lock") || p.getFileName().toString().startsWith("extra")) == false));
    Store targetStore = newStore(createTempDir(), false);
    MultiFileWriter multiFileWriter = new MultiFileWriter(targetStore, mock(RecoveryState.Index.class), "", logger, () -> {
    });
    RecoveryTargetHandler target = new TestRecoveryTargetHandler() {

        @Override
        public void writeFileChunk(StoreFileMetadata md, long position, BytesReference content, boolean lastChunk, int totalTranslogOps, ActionListener<Void> listener) {
            ActionListener.completeWith(listener, () -> {
                multiFileWriter.writeFileChunk(md, position, content, lastChunk);
                return null;
            });
        }
    };
    RecoverySourceHandler handler = new RecoverySourceHandler(null, new AsyncRecoveryTarget(target, recoveryExecutor), threadPool, request, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), between(1, 8), between(1, 8)) {

        @Override
        protected void failEngine(IOException cause) {
            assertFalse(failedEngine.get());
            failedEngine.set(true);
        }
    };
    SetOnce<Exception> sendFilesError = new SetOnce<>();
    CountDownLatch latch = new CountDownLatch(1);
    handler.sendFiles(store, metas.toArray(new StoreFileMetadata[0]), () -> 0, new LatchedActionListener<>(ActionListener.wrap(r -> sendFilesError.set(null), e -> sendFilesError.set(e)), latch));
    latch.await();
    assertThat(sendFilesError.get(), instanceOf(IOException.class));
    assertNotNull(ExceptionsHelper.unwrapCorruption(sendFilesError.get()));
    assertTrue(failedEngine.get());
    // ensure all chunk requests have been completed; otherwise some files on the target are left open.
    IOUtils.close(() -> terminate(threadPool), () -> threadPool = null);
    IOUtils.close(store, multiFileWriter, targetStore);
}