Search in sources :

Example 1 with ReplicationTracker

use of org.opensearch.index.seqno.ReplicationTracker in project OpenSearch by opensearch-project.

the class InternalEngineTests method testSeqNoAndCheckpoints.

public void testSeqNoAndCheckpoints() throws IOException, InterruptedException {
    final int opCount = randomIntBetween(1, 256);
    long primarySeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final String[] ids = new String[] { "1", "2", "3" };
    final Set<String> indexedIds = new HashSet<>();
    long localCheckpoint = SequenceNumbers.NO_OPS_PERFORMED;
    long replicaLocalCheckpoint = SequenceNumbers.NO_OPS_PERFORMED;
    final long globalCheckpoint;
    long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    IOUtils.close(store, engine);
    store = createStore();
    InternalEngine initialEngine = null;
    try {
        initialEngine = createEngine(defaultSettings, store, createTempDir(), newLogMergePolicy(), null);
        final ShardRouting primary = TestShardRouting.newShardRouting("test", shardId.id(), "node1", null, true, ShardRoutingState.STARTED, allocationId);
        final ShardRouting initializingReplica = TestShardRouting.newShardRouting(shardId, "node2", false, ShardRoutingState.INITIALIZING);
        ReplicationTracker gcpTracker = (ReplicationTracker) initialEngine.config().getGlobalCheckpointSupplier();
        gcpTracker.updateFromMaster(1L, new HashSet<>(Collections.singletonList(primary.allocationId().getId())), new IndexShardRoutingTable.Builder(shardId).addShard(primary).build());
        gcpTracker.activatePrimaryMode(primarySeqNo);
        if (defaultSettings.isSoftDeleteEnabled()) {
            final CountDownLatch countDownLatch = new CountDownLatch(1);
            gcpTracker.addPeerRecoveryRetentionLease(initializingReplica.currentNodeId(), SequenceNumbers.NO_OPS_PERFORMED, ActionListener.wrap(countDownLatch::countDown));
            countDownLatch.await();
        }
        gcpTracker.updateFromMaster(2L, new HashSet<>(Collections.singletonList(primary.allocationId().getId())), new IndexShardRoutingTable.Builder(shardId).addShard(primary).addShard(initializingReplica).build());
        gcpTracker.initiateTracking(initializingReplica.allocationId().getId());
        gcpTracker.markAllocationIdAsInSync(initializingReplica.allocationId().getId(), replicaLocalCheckpoint);
        final ShardRouting replica = initializingReplica.moveToStarted();
        gcpTracker.updateFromMaster(3L, new HashSet<>(Arrays.asList(primary.allocationId().getId(), replica.allocationId().getId())), new IndexShardRoutingTable.Builder(shardId).addShard(primary).addShard(replica).build());
        for (int op = 0; op < opCount; op++) {
            final String id;
            // mostly index, sometimes delete
            if (rarely() && indexedIds.isEmpty() == false) {
                // we have some docs indexed, so delete one of them
                id = randomFrom(indexedIds);
                final Engine.Delete delete = new Engine.Delete("test", id, newUid(id), UNASSIGNED_SEQ_NO, primaryTerm.get(), rarely() ? 100 : Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0);
                final Engine.DeleteResult result = initialEngine.delete(delete);
                if (result.getResultType() == Engine.Result.Type.SUCCESS) {
                    assertThat(result.getSeqNo(), equalTo(primarySeqNo + 1));
                    assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo + 1));
                    indexedIds.remove(id);
                    primarySeqNo++;
                } else {
                    assertThat(result.getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
                    assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo));
                }
            } else {
                // index a document
                id = randomFrom(ids);
                ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
                final Engine.Index index = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, primaryTerm.get(), rarely() ? 100 : Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
                final Engine.IndexResult result = initialEngine.index(index);
                if (result.getResultType() == Engine.Result.Type.SUCCESS) {
                    assertThat(result.getSeqNo(), equalTo(primarySeqNo + 1));
                    assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo + 1));
                    indexedIds.add(id);
                    primarySeqNo++;
                } else {
                    assertThat(result.getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
                    assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo));
                }
            }
            // to advance persisted local checkpoint
            initialEngine.syncTranslog();
            if (randomInt(10) < 3) {
                // only update rarely as we do it every doc
                replicaLocalCheckpoint = randomIntBetween(Math.toIntExact(replicaLocalCheckpoint), Math.toIntExact(primarySeqNo));
            }
            gcpTracker.updateLocalCheckpoint(primary.allocationId().getId(), initialEngine.getPersistedLocalCheckpoint());
            gcpTracker.updateLocalCheckpoint(replica.allocationId().getId(), replicaLocalCheckpoint);
            if (rarely()) {
                localCheckpoint = primarySeqNo;
                maxSeqNo = primarySeqNo;
                initialEngine.flush(true, true);
            }
        }
        logger.info("localcheckpoint {}, global {}", replicaLocalCheckpoint, primarySeqNo);
        globalCheckpoint = gcpTracker.getGlobalCheckpoint();
        assertEquals(primarySeqNo, initialEngine.getSeqNoStats(-1).getMaxSeqNo());
        assertEquals(primarySeqNo, initialEngine.getPersistedLocalCheckpoint());
        assertThat(globalCheckpoint, equalTo(replicaLocalCheckpoint));
        assertThat(Long.parseLong(initialEngine.commitStats().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)), equalTo(localCheckpoint));
        // to guarantee the global checkpoint is written to the translog checkpoint
        initialEngine.getTranslog().sync();
        assertThat(initialEngine.getTranslog().getLastSyncedGlobalCheckpoint(), equalTo(globalCheckpoint));
        assertThat(Long.parseLong(initialEngine.commitStats().getUserData().get(SequenceNumbers.MAX_SEQ_NO)), equalTo(maxSeqNo));
    } finally {
        IOUtils.close(initialEngine);
    }
    try (InternalEngine recoveringEngine = new InternalEngine(initialEngine.config())) {
        recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        assertEquals(primarySeqNo, recoveringEngine.getSeqNoStats(-1).getMaxSeqNo());
        assertThat(Long.parseLong(recoveringEngine.commitStats().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)), equalTo(primarySeqNo));
        assertThat(recoveringEngine.getTranslog().getLastSyncedGlobalCheckpoint(), equalTo(globalCheckpoint));
        assertThat(Long.parseLong(recoveringEngine.commitStats().getUserData().get(SequenceNumbers.MAX_SEQ_NO)), // we have assigned sequence numbers to should be in the commit
        equalTo(primarySeqNo));
        assertThat(recoveringEngine.getProcessedLocalCheckpoint(), equalTo(primarySeqNo));
        assertThat(recoveringEngine.getPersistedLocalCheckpoint(), equalTo(primarySeqNo));
        assertThat(recoveringEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo));
        assertThat(generateNewSeqNo(recoveringEngine), equalTo(primarySeqNo + 1));
    }
}
Also used : IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Matchers.containsString(org.hamcrest.Matchers.containsString) CountDownLatch(java.util.concurrent.CountDownLatch) LongPoint(org.apache.lucene.document.LongPoint) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) ShardRouting(org.opensearch.cluster.routing.ShardRouting) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting) HashSet(java.util.HashSet)

Example 2 with ReplicationTracker

use of org.opensearch.index.seqno.ReplicationTracker in project OpenSearch by opensearch-project.

the class NoOpEngineTests method testNoopAfterRegularEngine.

public void testNoopAfterRegularEngine() throws IOException {
    int docs = randomIntBetween(1, 10);
    ReplicationTracker tracker = (ReplicationTracker) engine.config().getGlobalCheckpointSupplier();
    ShardRouting routing = TestShardRouting.newShardRouting("test", shardId.id(), "node", null, true, ShardRoutingState.STARTED, allocationId);
    IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(shardId).addShard(routing).build();
    tracker.updateFromMaster(1L, Collections.singleton(allocationId.getId()), table);
    tracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED);
    for (int i = 0; i < docs; i++) {
        ParsedDocument doc = testParsedDocument("" + i, null, testDocumentWithTextField(), B_1, null);
        engine.index(indexForDoc(doc));
        tracker.updateLocalCheckpoint(allocationId.getId(), i);
    }
    engine.flush(true, true);
    long localCheckpoint = engine.getPersistedLocalCheckpoint();
    long maxSeqNo = engine.getSeqNoStats(100L).getMaxSeqNo();
    engine.close();
    final NoOpEngine noOpEngine = new NoOpEngine(noOpConfig(INDEX_SETTINGS, store, primaryTranslogDir, tracker));
    assertThat(noOpEngine.getPersistedLocalCheckpoint(), equalTo(localCheckpoint));
    assertThat(noOpEngine.getSeqNoStats(100L).getMaxSeqNo(), equalTo(maxSeqNo));
    try (Engine.IndexCommitRef ref = noOpEngine.acquireLastIndexCommit(false)) {
        try (IndexReader reader = DirectoryReader.open(ref.getIndexCommit())) {
            assertThat(reader.numDocs(), equalTo(docs));
        }
    }
    noOpEngine.close();
}
Also used : IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) IndexReader(org.apache.lucene.index.IndexReader) ShardRouting(org.opensearch.cluster.routing.ShardRouting) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting)

Example 3 with ReplicationTracker

use of org.opensearch.index.seqno.ReplicationTracker in project OpenSearch by opensearch-project.

the class IndexShard method resetEngineToGlobalCheckpoint.

/**
 * Rollback the current engine to the safe commit, then replay local translog up to the global checkpoint.
 */
void resetEngineToGlobalCheckpoint() throws IOException {
    assert Thread.holdsLock(mutex) == false : "resetting engine under mutex";
    assert getActiveOperationsCount() == OPERATIONS_BLOCKED : "resetting engine without blocking operations; active operations are [" + getActiveOperations() + ']';
    // persist the global checkpoint to disk
    sync();
    final SeqNoStats seqNoStats = seqNoStats();
    final TranslogStats translogStats = translogStats();
    // flush to make sure the latest commit, which will be opened by the read-only engine, includes all operations.
    flush(new FlushRequest().waitIfOngoing(true));
    SetOnce<Engine> newEngineReference = new SetOnce<>();
    final long globalCheckpoint = getLastKnownGlobalCheckpoint();
    assert globalCheckpoint == getLastSyncedGlobalCheckpoint();
    synchronized (engineMutex) {
        verifyNotClosed();
        // we must create both new read-only engine and new read-write engine under engineMutex to ensure snapshotStoreMetadata,
        // acquireXXXCommit and close works.
        final Engine readOnlyEngine = new ReadOnlyEngine(newEngineConfig(replicationTracker), seqNoStats, translogStats, false, Function.identity(), true) {

            @Override
            public IndexCommitRef acquireLastIndexCommit(boolean flushFirst) {
                synchronized (engineMutex) {
                    if (newEngineReference.get() == null) {
                        throw new AlreadyClosedException("engine was closed");
                    }
                    // ignore flushFirst since we flushed above and we do not want to interfere with ongoing translog replay
                    return newEngineReference.get().acquireLastIndexCommit(false);
                }
            }

            @Override
            public IndexCommitRef acquireSafeIndexCommit() {
                synchronized (engineMutex) {
                    if (newEngineReference.get() == null) {
                        throw new AlreadyClosedException("engine was closed");
                    }
                    return newEngineReference.get().acquireSafeIndexCommit();
                }
            }

            @Override
            public void close() throws IOException {
                assert Thread.holdsLock(engineMutex);
                Engine newEngine = newEngineReference.get();
                if (newEngine == currentEngineReference.get()) {
                    // we successfully installed the new engine so do not close it.
                    newEngine = null;
                }
                IOUtils.close(super::close, newEngine);
            }
        };
        IOUtils.close(currentEngineReference.getAndSet(readOnlyEngine));
        newEngineReference.set(engineFactory.newReadWriteEngine(newEngineConfig(replicationTracker)));
        onNewEngine(newEngineReference.get());
    }
    final Engine.TranslogRecoveryRunner translogRunner = (engine, snapshot) -> runTranslogRecovery(engine, snapshot, Engine.Operation.Origin.LOCAL_RESET, () -> {
    // TODO: add a dedicate recovery stats for the reset translog
    });
    newEngineReference.get().recoverFromTranslog(translogRunner, globalCheckpoint);
    newEngineReference.get().refresh("reset_engine");
    synchronized (engineMutex) {
        verifyNotClosed();
        IOUtils.close(currentEngineReference.getAndSet(newEngineReference.get()));
        // We set active because we are now writing operations to the engine; this way,
        // if we go idle after some time and become inactive, we still give sync'd flush a chance to run.
        active.set(true);
    }
    // time elapses after the engine is created above (pulling the config settings) until we set the engine reference, during
    // which settings changes could possibly have happened, so here we forcefully push any config changes to the new engine.
    onSettingsChanged();
}
Also used : Query(org.apache.lucene.search.Query) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) Arrays(java.util.Arrays) LongSupplier(java.util.function.LongSupplier) CheckedFunction(org.opensearch.common.CheckedFunction) SnapshotRecoverySource(org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource) SearchStats(org.opensearch.index.search.stats.SearchStats) MetadataSnapshot(org.opensearch.index.store.Store.MetadataSnapshot) Term(org.apache.lucene.index.Term) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) ReferenceManager(org.apache.lucene.search.ReferenceManager) DocumentMapperForType(org.opensearch.index.mapper.DocumentMapperForType) MergeStats(org.opensearch.index.merge.MergeStats) UsageTrackingQueryCachingPolicy(org.apache.lucene.search.UsageTrackingQueryCachingPolicy) WriteStateException(org.opensearch.gateway.WriteStateException) RecoveryState(org.opensearch.indices.recovery.RecoveryState) RefreshFailedEngineException(org.opensearch.index.engine.RefreshFailedEngineException) Map(java.util.Map) Lucene(org.opensearch.common.lucene.Lucene) ObjectLongMap(com.carrotsearch.hppc.ObjectLongMap) QueryCachingPolicy(org.apache.lucene.search.QueryCachingPolicy) ActionListener(org.opensearch.action.ActionListener) GetResult(org.opensearch.index.engine.Engine.GetResult) IndexModule(org.opensearch.index.IndexModule) Segment(org.opensearch.index.engine.Segment) EnumSet(java.util.EnumSet) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) ShardRequestCache(org.opensearch.index.cache.request.ShardRequestCache) Index(org.opensearch.index.Index) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) ShardBitsetFilterCache(org.opensearch.index.cache.bitset.ShardBitsetFilterCache) GetStats(org.opensearch.index.get.GetStats) StandardCharsets(java.nio.charset.StandardCharsets) Engine(org.opensearch.index.engine.Engine) ClosedByInterruptException(java.nio.channels.ClosedByInterruptException) CountDownLatch(java.util.concurrent.CountDownLatch) VersionType(org.opensearch.index.VersionType) Logger(org.apache.logging.log4j.Logger) EngineConfigFactory(org.opensearch.index.engine.EngineConfigFactory) CheckedRunnable(org.opensearch.common.CheckedRunnable) ThreadInterruptedException(org.apache.lucene.util.ThreadInterruptedException) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) IndexCommit(org.apache.lucene.index.IndexCommit) ShardFieldData(org.opensearch.index.fielddata.ShardFieldData) RepositoriesService(org.opensearch.repositories.RepositoriesService) CodecService(org.opensearch.index.codec.CodecService) FlushRequest(org.opensearch.action.admin.indices.flush.FlushRequest) MapperParsingException(org.opensearch.index.mapper.MapperParsingException) ThreadPool(org.opensearch.threadpool.ThreadPool) Releasable(org.opensearch.common.lease.Releasable) MeanMetric(org.opensearch.common.metrics.MeanMetric) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) RecoverySource(org.opensearch.cluster.routing.RecoverySource) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) FilterDirectoryReader(org.apache.lucene.index.FilterDirectoryReader) LegacyESVersion(org.opensearch.LegacyESVersion) Mapping(org.opensearch.index.mapper.Mapping) Booleans(org.opensearch.common.Booleans) BiConsumer(java.util.function.BiConsumer) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) RetentionLeaseStats(org.opensearch.index.seqno.RetentionLeaseStats) StreamSupport(java.util.stream.StreamSupport) CommitStats(org.opensearch.index.engine.CommitStats) EngineFactory(org.opensearch.index.engine.EngineFactory) CompletionStats(org.opensearch.search.suggest.completion.CompletionStats) SetOnce(org.apache.lucene.util.SetOnce) RecoveryFailedException(org.opensearch.indices.recovery.RecoveryFailedException) PendingReplicationActions(org.opensearch.action.support.replication.PendingReplicationActions) RETAIN_ALL(org.opensearch.index.seqno.RetentionLeaseActions.RETAIN_ALL) FlushStats(org.opensearch.index.flush.FlushStats) IOException(java.io.IOException) BytesStreamOutput(org.opensearch.common.io.stream.BytesStreamOutput) IndexService(org.opensearch.index.IndexService) XContentHelper(org.opensearch.common.xcontent.XContentHelper) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecoveryStats(org.opensearch.index.recovery.RecoveryStats) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) CounterMetric(org.opensearch.common.metrics.CounterMetric) ShardIndexWarmerService(org.opensearch.index.warmer.ShardIndexWarmerService) RetentionLeaseSyncer(org.opensearch.index.seqno.RetentionLeaseSyncer) SafeCommitInfo(org.opensearch.index.engine.SafeCommitInfo) EngineException(org.opensearch.index.engine.EngineException) ReadOnlyEngine(org.opensearch.index.engine.ReadOnlyEngine) IdFieldMapper(org.opensearch.index.mapper.IdFieldMapper) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) TimeoutException(java.util.concurrent.TimeoutException) OpenSearchException(org.opensearch.OpenSearchException) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) Releasables(org.opensearch.common.lease.Releasables) UpgradeRequest(org.opensearch.action.admin.indices.upgrade.post.UpgradeRequest) RunOnce(org.opensearch.common.util.concurrent.RunOnce) ForceMergeRequest(org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest) MapperService(org.opensearch.index.mapper.MapperService) RefreshStats(org.opensearch.index.refresh.RefreshStats) Locale(java.util.Locale) Assertions(org.opensearch.Assertions) SegmentsStats(org.opensearch.index.engine.SegmentsStats) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) IndicesClusterStateService(org.opensearch.indices.cluster.IndicesClusterStateService) CheckIndex(org.apache.lucene.index.CheckIndex) Sort(org.apache.lucene.search.Sort) DirectoryReader(org.apache.lucene.index.DirectoryReader) IndicesService(org.opensearch.indices.IndicesService) TranslogConfig(org.opensearch.index.translog.TranslogConfig) IndexingMemoryController(org.opensearch.indices.IndexingMemoryController) RestStatus(org.opensearch.rest.RestStatus) Store(org.opensearch.index.store.Store) Collectors(java.util.stream.Collectors) SegmentInfos(org.apache.lucene.index.SegmentInfos) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) WarmerStats(org.opensearch.index.warmer.WarmerStats) Objects(java.util.Objects) List(java.util.List) ResyncTask(org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask) LeafReader(org.apache.lucene.index.LeafReader) IndexSettings(org.opensearch.index.IndexSettings) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) Optional(java.util.Optional) BigArrays(org.opensearch.common.util.BigArrays) Uid(org.opensearch.index.mapper.Uid) TranslogStats(org.opensearch.index.translog.TranslogStats) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ShardSearchStats(org.opensearch.index.search.stats.ShardSearchStats) ActionRunnable(org.opensearch.action.ActionRunnable) SimilarityService(org.opensearch.index.similarity.SimilarityService) CheckedConsumer(org.opensearch.common.CheckedConsumer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) EngineConfig(org.opensearch.index.engine.EngineConfig) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) SourceToParse(org.opensearch.index.mapper.SourceToParse) AsyncIOProcessor(org.opensearch.common.util.concurrent.AsyncIOProcessor) PeerRecoveryTargetService(org.opensearch.indices.recovery.PeerRecoveryTargetService) Translog(org.opensearch.index.translog.Translog) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) StoreStats(org.opensearch.index.store.StoreStats) RetentionLease(org.opensearch.index.seqno.RetentionLease) PrintStream(java.io.PrintStream) OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) RecoveryTarget(org.opensearch.indices.recovery.RecoveryTarget) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) IndexCache(org.opensearch.index.cache.IndexCache) DocumentMapper(org.opensearch.index.mapper.DocumentMapper) RootObjectMapper(org.opensearch.index.mapper.RootObjectMapper) ShardRouting(org.opensearch.cluster.routing.ShardRouting) IOUtils(org.opensearch.core.internal.io.IOUtils) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) ShardGetService(org.opensearch.index.get.ShardGetService) CircuitBreakerService(org.opensearch.indices.breaker.CircuitBreakerService) Closeable(java.io.Closeable) TypeMissingException(org.opensearch.indices.TypeMissingException) Collections(java.util.Collections) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) FlushRequest(org.opensearch.action.admin.indices.flush.FlushRequest) SetOnce(org.apache.lucene.util.SetOnce) ReadOnlyEngine(org.opensearch.index.engine.ReadOnlyEngine) TranslogStats(org.opensearch.index.translog.TranslogStats) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Engine(org.opensearch.index.engine.Engine) ReadOnlyEngine(org.opensearch.index.engine.ReadOnlyEngine)

Example 4 with ReplicationTracker

use of org.opensearch.index.seqno.ReplicationTracker in project OpenSearch by opensearch-project.

the class IndexShard method newEngineConfig.

private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) {
    final Sort indexSort = indexSortSupplier.get();
    final Engine.Warmer warmer = reader -> {
        assert Thread.holdsLock(mutex) == false : "warming engine under mutex";
        assert reader != null;
        if (this.warmer != null) {
            this.warmer.warm(reader);
        }
    };
    return this.engineConfigFactory.newEngineConfig(shardId, threadPool, indexSettings, warmer, store, indexSettings.getMergePolicy(), mapperService != null ? mapperService.indexAnalyzer() : null, similarityService.similarity(mapperService), engineConfigFactory.newCodecServiceOrDefault(indexSettings, mapperService, logger, codecService), shardEventListener, indexCache != null ? indexCache.query() : null, cachingPolicy, translogConfig, IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()), Arrays.asList(refreshListeners, refreshPendingLocationListener), Collections.singletonList(new RefreshMetricUpdater(refreshMetric)), indexSort, circuitBreakerService, globalCheckpointSupplier, replicationTracker::getRetentionLeases, () -> getOperationPrimaryTerm(), tombstoneDocSupplier());
}
Also used : Query(org.apache.lucene.search.Query) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) Arrays(java.util.Arrays) LongSupplier(java.util.function.LongSupplier) CheckedFunction(org.opensearch.common.CheckedFunction) SnapshotRecoverySource(org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource) SearchStats(org.opensearch.index.search.stats.SearchStats) MetadataSnapshot(org.opensearch.index.store.Store.MetadataSnapshot) Term(org.apache.lucene.index.Term) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) ReferenceManager(org.apache.lucene.search.ReferenceManager) DocumentMapperForType(org.opensearch.index.mapper.DocumentMapperForType) MergeStats(org.opensearch.index.merge.MergeStats) UsageTrackingQueryCachingPolicy(org.apache.lucene.search.UsageTrackingQueryCachingPolicy) WriteStateException(org.opensearch.gateway.WriteStateException) RecoveryState(org.opensearch.indices.recovery.RecoveryState) RefreshFailedEngineException(org.opensearch.index.engine.RefreshFailedEngineException) Map(java.util.Map) Lucene(org.opensearch.common.lucene.Lucene) ObjectLongMap(com.carrotsearch.hppc.ObjectLongMap) QueryCachingPolicy(org.apache.lucene.search.QueryCachingPolicy) ActionListener(org.opensearch.action.ActionListener) GetResult(org.opensearch.index.engine.Engine.GetResult) IndexModule(org.opensearch.index.IndexModule) Segment(org.opensearch.index.engine.Segment) EnumSet(java.util.EnumSet) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) ShardRequestCache(org.opensearch.index.cache.request.ShardRequestCache) Index(org.opensearch.index.Index) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) ShardBitsetFilterCache(org.opensearch.index.cache.bitset.ShardBitsetFilterCache) GetStats(org.opensearch.index.get.GetStats) StandardCharsets(java.nio.charset.StandardCharsets) Engine(org.opensearch.index.engine.Engine) ClosedByInterruptException(java.nio.channels.ClosedByInterruptException) CountDownLatch(java.util.concurrent.CountDownLatch) VersionType(org.opensearch.index.VersionType) Logger(org.apache.logging.log4j.Logger) EngineConfigFactory(org.opensearch.index.engine.EngineConfigFactory) CheckedRunnable(org.opensearch.common.CheckedRunnable) ThreadInterruptedException(org.apache.lucene.util.ThreadInterruptedException) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) IndexCommit(org.apache.lucene.index.IndexCommit) ShardFieldData(org.opensearch.index.fielddata.ShardFieldData) RepositoriesService(org.opensearch.repositories.RepositoriesService) CodecService(org.opensearch.index.codec.CodecService) FlushRequest(org.opensearch.action.admin.indices.flush.FlushRequest) MapperParsingException(org.opensearch.index.mapper.MapperParsingException) ThreadPool(org.opensearch.threadpool.ThreadPool) Releasable(org.opensearch.common.lease.Releasable) MeanMetric(org.opensearch.common.metrics.MeanMetric) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) RecoverySource(org.opensearch.cluster.routing.RecoverySource) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) FilterDirectoryReader(org.apache.lucene.index.FilterDirectoryReader) LegacyESVersion(org.opensearch.LegacyESVersion) Mapping(org.opensearch.index.mapper.Mapping) Booleans(org.opensearch.common.Booleans) BiConsumer(java.util.function.BiConsumer) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) RetentionLeaseStats(org.opensearch.index.seqno.RetentionLeaseStats) StreamSupport(java.util.stream.StreamSupport) CommitStats(org.opensearch.index.engine.CommitStats) EngineFactory(org.opensearch.index.engine.EngineFactory) CompletionStats(org.opensearch.search.suggest.completion.CompletionStats) SetOnce(org.apache.lucene.util.SetOnce) RecoveryFailedException(org.opensearch.indices.recovery.RecoveryFailedException) PendingReplicationActions(org.opensearch.action.support.replication.PendingReplicationActions) RETAIN_ALL(org.opensearch.index.seqno.RetentionLeaseActions.RETAIN_ALL) FlushStats(org.opensearch.index.flush.FlushStats) IOException(java.io.IOException) BytesStreamOutput(org.opensearch.common.io.stream.BytesStreamOutput) IndexService(org.opensearch.index.IndexService) XContentHelper(org.opensearch.common.xcontent.XContentHelper) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecoveryStats(org.opensearch.index.recovery.RecoveryStats) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) CounterMetric(org.opensearch.common.metrics.CounterMetric) ShardIndexWarmerService(org.opensearch.index.warmer.ShardIndexWarmerService) RetentionLeaseSyncer(org.opensearch.index.seqno.RetentionLeaseSyncer) SafeCommitInfo(org.opensearch.index.engine.SafeCommitInfo) EngineException(org.opensearch.index.engine.EngineException) ReadOnlyEngine(org.opensearch.index.engine.ReadOnlyEngine) IdFieldMapper(org.opensearch.index.mapper.IdFieldMapper) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) TimeoutException(java.util.concurrent.TimeoutException) OpenSearchException(org.opensearch.OpenSearchException) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) Releasables(org.opensearch.common.lease.Releasables) UpgradeRequest(org.opensearch.action.admin.indices.upgrade.post.UpgradeRequest) RunOnce(org.opensearch.common.util.concurrent.RunOnce) ForceMergeRequest(org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest) MapperService(org.opensearch.index.mapper.MapperService) RefreshStats(org.opensearch.index.refresh.RefreshStats) Locale(java.util.Locale) Assertions(org.opensearch.Assertions) SegmentsStats(org.opensearch.index.engine.SegmentsStats) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) IndicesClusterStateService(org.opensearch.indices.cluster.IndicesClusterStateService) CheckIndex(org.apache.lucene.index.CheckIndex) Sort(org.apache.lucene.search.Sort) DirectoryReader(org.apache.lucene.index.DirectoryReader) IndicesService(org.opensearch.indices.IndicesService) TranslogConfig(org.opensearch.index.translog.TranslogConfig) IndexingMemoryController(org.opensearch.indices.IndexingMemoryController) RestStatus(org.opensearch.rest.RestStatus) Store(org.opensearch.index.store.Store) Collectors(java.util.stream.Collectors) SegmentInfos(org.apache.lucene.index.SegmentInfos) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) WarmerStats(org.opensearch.index.warmer.WarmerStats) Objects(java.util.Objects) List(java.util.List) ResyncTask(org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask) LeafReader(org.apache.lucene.index.LeafReader) IndexSettings(org.opensearch.index.IndexSettings) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) Optional(java.util.Optional) BigArrays(org.opensearch.common.util.BigArrays) Uid(org.opensearch.index.mapper.Uid) TranslogStats(org.opensearch.index.translog.TranslogStats) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ShardSearchStats(org.opensearch.index.search.stats.ShardSearchStats) ActionRunnable(org.opensearch.action.ActionRunnable) SimilarityService(org.opensearch.index.similarity.SimilarityService) CheckedConsumer(org.opensearch.common.CheckedConsumer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) EngineConfig(org.opensearch.index.engine.EngineConfig) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) SourceToParse(org.opensearch.index.mapper.SourceToParse) AsyncIOProcessor(org.opensearch.common.util.concurrent.AsyncIOProcessor) PeerRecoveryTargetService(org.opensearch.indices.recovery.PeerRecoveryTargetService) Translog(org.opensearch.index.translog.Translog) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) StoreStats(org.opensearch.index.store.StoreStats) RetentionLease(org.opensearch.index.seqno.RetentionLease) PrintStream(java.io.PrintStream) OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) RecoveryTarget(org.opensearch.indices.recovery.RecoveryTarget) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) IndexCache(org.opensearch.index.cache.IndexCache) DocumentMapper(org.opensearch.index.mapper.DocumentMapper) RootObjectMapper(org.opensearch.index.mapper.RootObjectMapper) ShardRouting(org.opensearch.cluster.routing.ShardRouting) IOUtils(org.opensearch.core.internal.io.IOUtils) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) ShardGetService(org.opensearch.index.get.ShardGetService) CircuitBreakerService(org.opensearch.indices.breaker.CircuitBreakerService) Closeable(java.io.Closeable) TypeMissingException(org.opensearch.indices.TypeMissingException) Collections(java.util.Collections) Sort(org.apache.lucene.search.Sort) Engine(org.opensearch.index.engine.Engine) ReadOnlyEngine(org.opensearch.index.engine.ReadOnlyEngine)

Example 5 with ReplicationTracker

use of org.opensearch.index.seqno.ReplicationTracker in project OpenSearch by opensearch-project.

the class EngineTestCase method config.

public EngineConfig config(final IndexSettings indexSettings, final Store store, final Path translogPath, final MergePolicy mergePolicy, final ReferenceManager.RefreshListener externalRefreshListener, final ReferenceManager.RefreshListener internalRefreshListener, final Sort indexSort, @Nullable final LongSupplier maybeGlobalCheckpointSupplier, @Nullable final Supplier<RetentionLeases> maybeRetentionLeasesSupplier, final CircuitBreakerService breakerService) {
    final IndexWriterConfig iwc = newIndexWriterConfig();
    final TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    final Engine.EventListener eventListener = new Engine.EventListener() {
    };
    // we don't need to notify anybody in this test
    final List<ReferenceManager.RefreshListener> extRefreshListenerList = externalRefreshListener == null ? emptyList() : Collections.singletonList(externalRefreshListener);
    final List<ReferenceManager.RefreshListener> intRefreshListenerList = internalRefreshListener == null ? emptyList() : Collections.singletonList(internalRefreshListener);
    final LongSupplier globalCheckpointSupplier;
    final Supplier<RetentionLeases> retentionLeasesSupplier;
    if (maybeGlobalCheckpointSupplier == null) {
        assert maybeRetentionLeasesSupplier == null;
        final ReplicationTracker replicationTracker = new ReplicationTracker(shardId, allocationId.getId(), indexSettings, randomNonNegativeLong(), SequenceNumbers.NO_OPS_PERFORMED, update -> {
        }, () -> 0L, (leases, listener) -> listener.onResponse(new ReplicationResponse()), () -> SafeCommitInfo.EMPTY);
        globalCheckpointSupplier = replicationTracker;
        retentionLeasesSupplier = replicationTracker::getRetentionLeases;
    } else {
        assert maybeRetentionLeasesSupplier != null;
        globalCheckpointSupplier = maybeGlobalCheckpointSupplier;
        retentionLeasesSupplier = maybeRetentionLeasesSupplier;
    }
    return new EngineConfig(shardId, threadPool, indexSettings, null, store, mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), extRefreshListenerList, intRefreshListenerList, indexSort, breakerService, globalCheckpointSupplier, retentionLeasesSupplier, primaryTerm, tombstoneDocSupplier());
}
Also used : TranslogConfig(org.opensearch.index.translog.TranslogConfig) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) CodecService(org.opensearch.index.codec.CodecService) LongSupplier(java.util.function.LongSupplier) LiveIndexWriterConfig(org.apache.lucene.index.LiveIndexWriterConfig) IndexWriterConfig(org.apache.lucene.index.IndexWriterConfig)

Aggregations

ReplicationTracker (org.opensearch.index.seqno.ReplicationTracker)6 IndexShardRoutingTable (org.opensearch.cluster.routing.IndexShardRoutingTable)5 ShardRouting (org.opensearch.cluster.routing.ShardRouting)5 HashSet (java.util.HashSet)3 CountDownLatch (java.util.concurrent.CountDownLatch)3 LongSupplier (java.util.function.LongSupplier)3 ParsedDocument (org.opensearch.index.mapper.ParsedDocument)3 ObjectLongMap (com.carrotsearch.hppc.ObjectLongMap)2 Closeable (java.io.Closeable)2 IOException (java.io.IOException)2 PrintStream (java.io.PrintStream)2 ClosedByInterruptException (java.nio.channels.ClosedByInterruptException)2 StandardCharsets (java.nio.charset.StandardCharsets)2 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 Collections (java.util.Collections)2 EnumSet (java.util.EnumSet)2 List (java.util.List)2 Locale (java.util.Locale)2 Map (java.util.Map)2