Search in sources :

Example 31 with AllocationId

use of org.opensearch.cluster.routing.AllocationId in project OpenSearch by opensearch-project.

the class IndexShardTests method testRelocatedShardCanNotBeRevivedConcurrently.

public void testRelocatedShardCanNotBeRevivedConcurrently() throws IOException, InterruptedException, BrokenBarrierException {
    final IndexShard shard = newStartedShard(true);
    final ShardRouting originalRouting = shard.routingEntry();
    final ShardRouting relocationRouting = ShardRoutingHelper.relocate(originalRouting, "other_node");
    IndexShardTestCase.updateRoutingEntry(shard, relocationRouting);
    CyclicBarrier cyclicBarrier = new CyclicBarrier(3);
    AtomicReference<Exception> relocationException = new AtomicReference<>();
    Thread relocationThread = new Thread(new AbstractRunnable() {

        @Override
        public void onFailure(Exception e) {
            relocationException.set(e);
        }

        @Override
        protected void doRun() throws Exception {
            cyclicBarrier.await();
            shard.relocated(relocationRouting.getTargetRelocatingShard().allocationId().getId(), primaryContext -> {
            });
        }
    });
    relocationThread.start();
    AtomicReference<Exception> cancellingException = new AtomicReference<>();
    Thread cancellingThread = new Thread(new AbstractRunnable() {

        @Override
        public void onFailure(Exception e) {
            cancellingException.set(e);
        }

        @Override
        protected void doRun() throws Exception {
            cyclicBarrier.await();
            IndexShardTestCase.updateRoutingEntry(shard, originalRouting);
        }
    });
    cancellingThread.start();
    cyclicBarrier.await();
    relocationThread.join();
    cancellingThread.join();
    if (shard.isRelocatedPrimary()) {
        logger.debug("shard was relocated successfully");
        assertThat(cancellingException.get(), instanceOf(IllegalIndexShardStateException.class));
        assertThat("current routing:" + shard.routingEntry(), shard.routingEntry().relocating(), equalTo(true));
        assertThat(relocationException.get(), nullValue());
    } else {
        logger.debug("shard relocation was cancelled");
        assertThat(relocationException.get(), either(instanceOf(IllegalIndexShardStateException.class)).or(instanceOf(IllegalStateException.class)));
        assertThat("current routing:" + shard.routingEntry(), shard.routingEntry().relocating(), equalTo(false));
        assertThat(cancellingException.get(), nullValue());
    }
    closeShards(shard);
}
Also used : AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) Matchers.hasToString(org.hamcrest.Matchers.hasToString) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) MockFSDirectoryFactory(org.opensearch.test.store.MockFSDirectoryFactory) Arrays(java.util.Arrays) CheckedFunction(org.opensearch.common.CheckedFunction) IndexScopedSettings(org.opensearch.common.settings.IndexScopedSettings) Term(org.apache.lucene.index.Term) Matchers.not(org.hamcrest.Matchers.not) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Version(org.opensearch.Version) Strings(org.opensearch.common.Strings) InternalEngine(org.opensearch.index.engine.InternalEngine) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) IndexFieldDataCache(org.opensearch.index.fielddata.IndexFieldDataCache) RecoveryState(org.opensearch.indices.recovery.RecoveryState) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) ActionListener(org.opensearch.action.ActionListener) IOContext(org.apache.lucene.store.IOContext) Path(java.nio.file.Path) NodeEnvironment(org.opensearch.env.NodeEnvironment) TimeValue(org.opensearch.common.unit.TimeValue) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) RegexMatcher.matches(org.opensearch.test.hamcrest.RegexMatcher.matches) Engine(org.opensearch.index.engine.Engine) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) FieldMaskingReader(org.opensearch.test.FieldMaskingReader) FileVisitResult(java.nio.file.FileVisitResult) CountDownLatch(java.util.concurrent.CountDownLatch) VersionType(org.opensearch.index.VersionType) Logger(org.apache.logging.log4j.Logger) EngineConfigFactory(org.opensearch.index.engine.EngineConfigFactory) Stream(java.util.stream.Stream) Randomness(org.opensearch.common.Randomness) BytesArray(org.opensearch.common.bytes.BytesArray) XContentType(org.opensearch.common.xcontent.XContentType) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Matchers.containsString(org.hamcrest.Matchers.containsString) Matchers.in(org.hamcrest.Matchers.in) XContentFactory.jsonBuilder(org.opensearch.common.xcontent.XContentFactory.jsonBuilder) CodecService(org.opensearch.index.codec.CodecService) FlushRequest(org.opensearch.action.admin.indices.flush.FlushRequest) MapperParsingException(org.opensearch.index.mapper.MapperParsingException) ThreadPool(org.opensearch.threadpool.ThreadPool) TestShardRouting.newShardRouting(org.opensearch.cluster.routing.TestShardRouting.newShardRouting) Releasable(org.opensearch.common.lease.Releasable) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) RecoverySource(org.opensearch.cluster.routing.RecoverySource) DocIdSeqNoAndSource(org.opensearch.index.engine.DocIdSeqNoAndSource) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) VersionUtils(org.opensearch.test.VersionUtils) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) VersionFieldMapper(org.opensearch.index.mapper.VersionFieldMapper) Matchers.hasSize(org.hamcrest.Matchers.hasSize) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) CorruptionUtils(org.opensearch.test.CorruptionUtils) CommitStats(org.opensearch.index.engine.CommitStats) TopDocs(org.apache.lucene.search.TopDocs) ParseContext(org.opensearch.index.mapper.ParseContext) Versions(org.opensearch.common.lucene.uid.Versions) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) Files(java.nio.file.Files) TestTranslog(org.opensearch.index.translog.TestTranslog) IOException(java.io.IOException) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) BytesStreamOutput(org.opensearch.common.io.stream.BytesStreamOutput) SourceFieldMapper(org.opensearch.index.mapper.SourceFieldMapper) IndexFieldDataService(org.opensearch.index.fielddata.IndexFieldDataService) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Matchers.sameInstance(org.hamcrest.Matchers.sameInstance) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) RetentionLeaseSyncer(org.opensearch.index.seqno.RetentionLeaseSyncer) Assert(org.junit.Assert) SeqNoFieldMapper(org.opensearch.index.mapper.SeqNoFieldMapper) CommonStats(org.opensearch.action.admin.indices.stats.CommonStats) ReadOnlyEngine(org.opensearch.index.engine.ReadOnlyEngine) IdFieldMapper(org.opensearch.index.mapper.IdFieldMapper) Matchers.either(org.hamcrest.Matchers.either) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) IndexableField(org.apache.lucene.index.IndexableField) Lucene.cleanLuceneIndex(org.opensearch.common.lucene.Lucene.cleanLuceneIndex) NoneCircuitBreakerService(org.opensearch.indices.breaker.NoneCircuitBreakerService) OpenSearchException(org.opensearch.OpenSearchException) Releasables(org.opensearch.common.lease.Releasables) CommonStatsFlags(org.opensearch.action.admin.indices.stats.CommonStatsFlags) Matchers.hasKey(org.hamcrest.Matchers.hasKey) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) ForceMergeRequest(org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) MapperService(org.opensearch.index.mapper.MapperService) IndexId(org.opensearch.repositories.IndexId) Matchers.everyItem(org.hamcrest.Matchers.everyItem) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) Directory(org.apache.lucene.store.Directory) Assertions(org.opensearch.Assertions) XContentFactory(org.opensearch.common.xcontent.XContentFactory) DummyShardLock(org.opensearch.test.DummyShardLock) UnassignedInfo(org.opensearch.cluster.routing.UnassignedInfo) SimpleFileVisitor(java.nio.file.SimpleFileVisitor) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) EngineTestCase(org.opensearch.index.engine.EngineTestCase) CyclicBarrier(java.util.concurrent.CyclicBarrier) DeleteResult(org.opensearch.index.engine.Engine.DeleteResult) BytesRef(org.apache.lucene.util.BytesRef) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) SnapshotId(org.opensearch.snapshots.SnapshotId) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) DirectoryReader(org.apache.lucene.index.DirectoryReader) Store(org.opensearch.index.store.Store) Collectors(java.util.stream.Collectors) Tuple(org.opensearch.common.collect.Tuple) List(java.util.List) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Matchers.equalTo(org.hamcrest.Matchers.equalTo) IndexSettings(org.opensearch.index.IndexSettings) ShardRoutingHelper(org.opensearch.cluster.routing.ShardRoutingHelper) Uid(org.opensearch.index.mapper.Uid) TranslogStats(org.opensearch.index.translog.TranslogStats) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) IntStream(java.util.stream.IntStream) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) EngineConfig(org.opensearch.index.engine.EngineConfig) StoreUtils(org.opensearch.index.store.StoreUtils) IndicesFieldDataCache(org.opensearch.indices.fielddata.cache.IndicesFieldDataCache) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) IndexFieldData(org.opensearch.index.fielddata.IndexFieldData) SourceToParse(org.opensearch.index.mapper.SourceToParse) Charset(java.nio.charset.Charset) Translog(org.opensearch.index.translog.Translog) UUIDs(org.opensearch.common.UUIDs) IndicesQueryCache(org.opensearch.indices.IndicesQueryCache) StoreStats(org.opensearch.index.store.StoreStats) RetentionLease(org.opensearch.index.seqno.RetentionLease) StreamInput(org.opensearch.common.io.stream.StreamInput) InternalEngineFactory(org.opensearch.index.engine.InternalEngineFactory) Collections.emptyMap(java.util.Collections.emptyMap) Matchers.oneOf(org.hamcrest.Matchers.oneOf) RecoveryTarget(org.opensearch.indices.recovery.RecoveryTarget) LongFunction(java.util.function.LongFunction) Collections.emptySet(java.util.Collections.emptySet) AllocationId(org.opensearch.cluster.routing.AllocationId) Semaphore(java.util.concurrent.Semaphore) BasicFileAttributes(java.nio.file.attribute.BasicFileAttributes) EMPTY_PARAMS(org.opensearch.common.xcontent.ToXContent.EMPTY_PARAMS) ShardRouting(org.opensearch.cluster.routing.ShardRouting) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting) TermQuery(org.apache.lucene.search.TermQuery) Constants(org.apache.lucene.util.Constants) AtomicArray(org.opensearch.common.util.concurrent.AtomicArray) FilterDirectory(org.apache.lucene.store.FilterDirectory) Snapshot(org.opensearch.snapshots.Snapshot) IndexRequest(org.opensearch.action.index.IndexRequest) Collections(java.util.Collections) AtomicReference(java.util.concurrent.atomic.AtomicReference) TestShardRouting.newShardRouting(org.opensearch.cluster.routing.TestShardRouting.newShardRouting) ShardRouting(org.opensearch.cluster.routing.ShardRouting) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) MapperParsingException(org.opensearch.index.mapper.MapperParsingException) IOException(java.io.IOException) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) ExecutionException(java.util.concurrent.ExecutionException) OpenSearchException(org.opensearch.OpenSearchException) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) CyclicBarrier(java.util.concurrent.CyclicBarrier)

Example 32 with AllocationId

use of org.opensearch.cluster.routing.AllocationId in project OpenSearch by opensearch-project.

the class IndexShardTests method testShardStateMetaHashCodeEquals.

public void testShardStateMetaHashCodeEquals() {
    AllocationId allocationId = randomBoolean() ? null : randomAllocationId();
    ShardStateMetadata meta = new ShardStateMetadata(randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10), allocationId);
    assertEquals(meta, new ShardStateMetadata(meta.primary, meta.indexUUID, meta.allocationId));
    assertEquals(meta.hashCode(), new ShardStateMetadata(meta.primary, meta.indexUUID, meta.allocationId).hashCode());
    assertFalse(meta.equals(new ShardStateMetadata(!meta.primary, meta.indexUUID, meta.allocationId)));
    assertFalse(meta.equals(new ShardStateMetadata(!meta.primary, meta.indexUUID + "foo", meta.allocationId)));
    assertFalse(meta.equals(new ShardStateMetadata(!meta.primary, meta.indexUUID + "foo", randomAllocationId())));
    Set<Integer> hashCodes = new HashSet<>();
    for (int i = 0; i < 30; i++) {
        // just a sanity check that we impl hashcode
        allocationId = randomBoolean() ? null : randomAllocationId();
        meta = new ShardStateMetadata(randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10), allocationId);
        hashCodes.add(meta.hashCode());
    }
    assertTrue("more than one unique hashcode expected but got: " + hashCodes.size(), hashCodes.size() > 1);
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) AllocationId(org.opensearch.cluster.routing.AllocationId) HashSet(java.util.HashSet)

Example 33 with AllocationId

use of org.opensearch.cluster.routing.AllocationId in project OpenSearch by opensearch-project.

the class ReplicationTrackerRetentionLeaseTests method testReplicaIgnoresOlderRetentionLeasesVersion.

public void testReplicaIgnoresOlderRetentionLeasesVersion() {
    final AllocationId allocationId = AllocationId.newInitializing();
    final ReplicationTracker replicationTracker = new ReplicationTracker(new ShardId("test", "_na", 0), allocationId.getId(), IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), randomNonNegativeLong(), UNASSIGNED_SEQ_NO, value -> {
    }, () -> 0L, (leases, listener) -> {
    }, OPS_BASED_RECOVERY_ALWAYS_REASONABLE);
    replicationTracker.updateFromMaster(randomNonNegativeLong(), Collections.singleton(allocationId.getId()), routingTable(Collections.emptySet(), allocationId));
    final int length = randomIntBetween(0, 8);
    final List<RetentionLeases> retentionLeasesCollection = new ArrayList<>(length);
    long primaryTerm = 1;
    long version = 0;
    for (int i = 0; i < length; i++) {
        final int innerLength = randomIntBetween(0, 8);
        final Collection<RetentionLease> leases = new ArrayList<>();
        for (int j = 0; j < innerLength; j++) {
            leases.add(new RetentionLease(i + "-" + j, randomNonNegativeLong(), randomNonNegativeLong(), randomAlphaOfLength(8)));
        }
        version++;
        if (rarely()) {
            primaryTerm++;
        }
        retentionLeasesCollection.add(new RetentionLeases(primaryTerm, version, leases));
    }
    final Collection<RetentionLease> expectedLeases;
    if (length == 0 || retentionLeasesCollection.get(length - 1).leases().isEmpty()) {
        expectedLeases = Collections.emptyList();
    } else {
        expectedLeases = retentionLeasesCollection.get(length - 1).leases();
    }
    Collections.shuffle(retentionLeasesCollection, random());
    for (final RetentionLeases retentionLeases : retentionLeasesCollection) {
        replicationTracker.updateRetentionLeasesOnReplica(retentionLeases);
    }
    assertThat(replicationTracker.getRetentionLeases().version(), equalTo(version));
    if (expectedLeases.isEmpty()) {
        assertThat(replicationTracker.getRetentionLeases().leases(), empty());
    } else {
        assertThat(replicationTracker.getRetentionLeases().leases(), contains(expectedLeases.toArray(new RetentionLease[0])));
    }
}
Also used : ShardId(org.opensearch.index.shard.ShardId) AllocationId(org.opensearch.cluster.routing.AllocationId) ArrayList(java.util.ArrayList)

Example 34 with AllocationId

use of org.opensearch.cluster.routing.AllocationId in project OpenSearch by opensearch-project.

the class ReplicationTrackerRetentionLeaseTests method testRemoveRetentionLease.

public void testRemoveRetentionLease() {
    final AllocationId allocationId = AllocationId.newInitializing();
    long primaryTerm = randomLongBetween(1, Long.MAX_VALUE);
    final ReplicationTracker replicationTracker = new ReplicationTracker(new ShardId("test", "_na", 0), allocationId.getId(), IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), primaryTerm, UNASSIGNED_SEQ_NO, value -> {
    }, () -> 0L, (leases, listener) -> {
    }, OPS_BASED_RECOVERY_ALWAYS_REASONABLE);
    replicationTracker.updateFromMaster(randomNonNegativeLong(), Collections.singleton(allocationId.getId()), routingTable(Collections.emptySet(), allocationId));
    replicationTracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED);
    final int length = randomIntBetween(0, 8);
    final long[] minimumRetainingSequenceNumbers = new long[length];
    for (int i = 0; i < length; i++) {
        if (rarely() && primaryTerm < Long.MAX_VALUE) {
            primaryTerm = randomLongBetween(primaryTerm + 1, Long.MAX_VALUE);
            replicationTracker.setOperationPrimaryTerm(primaryTerm);
        }
        minimumRetainingSequenceNumbers[i] = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
        replicationTracker.addRetentionLease(Integer.toString(i), minimumRetainingSequenceNumbers[i], "test-" + i, ActionListener.wrap(() -> {
        }));
    }
    for (int i = 0; i < length; i++) {
        if (rarely() && primaryTerm < Long.MAX_VALUE) {
            primaryTerm = randomLongBetween(primaryTerm + 1, Long.MAX_VALUE);
            replicationTracker.setOperationPrimaryTerm(primaryTerm);
        }
        /*
             * Remove from the end since it will make the following assertion easier; we want to ensure that only the intended lease was
             * removed.
             */
        replicationTracker.removeRetentionLease(Integer.toString(length - i - 1), ActionListener.wrap(() -> {
        }));
        assertRetentionLeases(replicationTracker, length - i - 1, minimumRetainingSequenceNumbers, primaryTerm, 2 + length + i, true, false);
    }
}
Also used : ShardId(org.opensearch.index.shard.ShardId) AllocationId(org.opensearch.cluster.routing.AllocationId)

Example 35 with AllocationId

use of org.opensearch.cluster.routing.AllocationId in project OpenSearch by opensearch-project.

the class ReplicationTrackerRetentionLeaseTests method testLoadAndPersistRetentionLeases.

public void testLoadAndPersistRetentionLeases() throws IOException {
    final AllocationId allocationId = AllocationId.newInitializing();
    long primaryTerm = randomLongBetween(1, Long.MAX_VALUE);
    final ReplicationTracker replicationTracker = new ReplicationTracker(new ShardId("test", "_na", 0), allocationId.getId(), IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), primaryTerm, UNASSIGNED_SEQ_NO, value -> {
    }, () -> 0L, (leases, listener) -> {
    }, OPS_BASED_RECOVERY_ALWAYS_REASONABLE);
    replicationTracker.updateFromMaster(randomNonNegativeLong(), Collections.singleton(allocationId.getId()), routingTable(Collections.emptySet(), allocationId));
    replicationTracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED);
    final int length = randomIntBetween(0, 8);
    for (int i = 0; i < length; i++) {
        if (rarely() && primaryTerm < Long.MAX_VALUE) {
            primaryTerm = randomLongBetween(primaryTerm + 1, Long.MAX_VALUE);
            replicationTracker.setOperationPrimaryTerm(primaryTerm);
        }
        final long retainingSequenceNumber = randomLongBetween(SequenceNumbers.NO_OPS_PERFORMED, Long.MAX_VALUE);
        replicationTracker.addRetentionLease(Integer.toString(i), retainingSequenceNumber, "test-" + i, ActionListener.wrap(() -> {
        }));
    }
    final Path path = createTempDir();
    replicationTracker.persistRetentionLeases(path);
    assertThat(replicationTracker.loadRetentionLeases(path), equalTo(replicationTracker.getRetentionLeases()));
}
Also used : ShardId(org.opensearch.index.shard.ShardId) Path(java.nio.file.Path) AllocationId(org.opensearch.cluster.routing.AllocationId)

Aggregations

AllocationId (org.opensearch.cluster.routing.AllocationId)47 ShardId (org.opensearch.index.shard.ShardId)29 AtomicLong (java.util.concurrent.atomic.AtomicLong)21 IndexShardRoutingTable (org.opensearch.cluster.routing.IndexShardRoutingTable)17 HashSet (java.util.HashSet)16 BrokenBarrierException (java.util.concurrent.BrokenBarrierException)16 CyclicBarrier (java.util.concurrent.CyclicBarrier)16 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)16 ShardRouting (org.opensearch.cluster.routing.ShardRouting)16 TestShardRouting (org.opensearch.cluster.routing.TestShardRouting)14 ArrayList (java.util.ArrayList)13 Set (java.util.Set)13 Matchers.containsString (org.hamcrest.Matchers.containsString)13 Matchers.hasToString (org.hamcrest.Matchers.hasToString)13 ShardRoutingState (org.opensearch.cluster.routing.ShardRoutingState)13 Settings (org.opensearch.common.settings.Settings)13 IndexSettings (org.opensearch.index.IndexSettings)13 IOException (java.io.IOException)12 Collections (java.util.Collections)12 Collections.emptySet (java.util.Collections.emptySet)12