Examples with ShardRouting org.opensearch.cluster.routing.ShardRouting used on opensource projects

Example 61 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class TestGatewayAllocator method applyFailedShards.

@Override
public void applyFailedShards(List<FailedShard> failedShards, RoutingAllocation allocation) {
    currentNodes = allocation.nodes();
    for (FailedShard failedShard : failedShards) {
        final ShardRouting failedRouting = failedShard.getRoutingEntry();
        Map<ShardId, ShardRouting> nodeAllocations = knownAllocations.get(failedRouting.currentNodeId());
        if (nodeAllocations != null) {
            nodeAllocations.remove(failedRouting.shardId());
            if (nodeAllocations.isEmpty()) {
                knownAllocations.remove(failedRouting.currentNodeId());
            }
        }
    }
}

Example 62 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class InternalEngineTests method testSeqNoAndCheckpoints.

public void testSeqNoAndCheckpoints() throws IOException, InterruptedException {
    final int opCount = randomIntBetween(1, 256);
    long primarySeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    final String[] ids = new String[] { "1", "2", "3" };
    final Set<String> indexedIds = new HashSet<>();
    long localCheckpoint = SequenceNumbers.NO_OPS_PERFORMED;
    long replicaLocalCheckpoint = SequenceNumbers.NO_OPS_PERFORMED;
    final long globalCheckpoint;
    long maxSeqNo = SequenceNumbers.NO_OPS_PERFORMED;
    IOUtils.close(store, engine);
    store = createStore();
    InternalEngine initialEngine = null;
    try {
        initialEngine = createEngine(defaultSettings, store, createTempDir(), newLogMergePolicy(), null);
        final ShardRouting primary = TestShardRouting.newShardRouting("test", shardId.id(), "node1", null, true, ShardRoutingState.STARTED, allocationId);
        final ShardRouting initializingReplica = TestShardRouting.newShardRouting(shardId, "node2", false, ShardRoutingState.INITIALIZING);
        ReplicationTracker gcpTracker = (ReplicationTracker) initialEngine.config().getGlobalCheckpointSupplier();
        gcpTracker.updateFromMaster(1L, new HashSet<>(Collections.singletonList(primary.allocationId().getId())), new IndexShardRoutingTable.Builder(shardId).addShard(primary).build());
        gcpTracker.activatePrimaryMode(primarySeqNo);
        if (defaultSettings.isSoftDeleteEnabled()) {
            final CountDownLatch countDownLatch = new CountDownLatch(1);
            gcpTracker.addPeerRecoveryRetentionLease(initializingReplica.currentNodeId(), SequenceNumbers.NO_OPS_PERFORMED, ActionListener.wrap(countDownLatch::countDown));
            countDownLatch.await();
        }
        gcpTracker.updateFromMaster(2L, new HashSet<>(Collections.singletonList(primary.allocationId().getId())), new IndexShardRoutingTable.Builder(shardId).addShard(primary).addShard(initializingReplica).build());
        gcpTracker.initiateTracking(initializingReplica.allocationId().getId());
        gcpTracker.markAllocationIdAsInSync(initializingReplica.allocationId().getId(), replicaLocalCheckpoint);
        final ShardRouting replica = initializingReplica.moveToStarted();
        gcpTracker.updateFromMaster(3L, new HashSet<>(Arrays.asList(primary.allocationId().getId(), replica.allocationId().getId())), new IndexShardRoutingTable.Builder(shardId).addShard(primary).addShard(replica).build());
        for (int op = 0; op < opCount; op++) {
            final String id;
            // mostly index, sometimes delete
            if (rarely() && indexedIds.isEmpty() == false) {
                // we have some docs indexed, so delete one of them
                id = randomFrom(indexedIds);
                final Engine.Delete delete = new Engine.Delete("test", id, newUid(id), UNASSIGNED_SEQ_NO, primaryTerm.get(), rarely() ? 100 : Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), UNASSIGNED_SEQ_NO, 0);
                final Engine.DeleteResult result = initialEngine.delete(delete);
                if (result.getResultType() == Engine.Result.Type.SUCCESS) {
                    assertThat(result.getSeqNo(), equalTo(primarySeqNo + 1));
                    assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo + 1));
                    indexedIds.remove(id);
                    primarySeqNo++;
                } else {
                    assertThat(result.getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
                    assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo));
                }
            } else {
                // index a document
                id = randomFrom(ids);
                ParsedDocument doc = testParsedDocument(id, null, testDocumentWithTextField(), SOURCE, null);
                final Engine.Index index = new Engine.Index(newUid(doc), doc, UNASSIGNED_SEQ_NO, primaryTerm.get(), rarely() ? 100 : Versions.MATCH_ANY, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false, UNASSIGNED_SEQ_NO, 0);
                final Engine.IndexResult result = initialEngine.index(index);
                if (result.getResultType() == Engine.Result.Type.SUCCESS) {
                    assertThat(result.getSeqNo(), equalTo(primarySeqNo + 1));
                    assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo + 1));
                    indexedIds.add(id);
                    primarySeqNo++;
                } else {
                    assertThat(result.getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
                    assertThat(initialEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo));
                }
            }
            // to advance persisted local checkpoint
            initialEngine.syncTranslog();
            if (randomInt(10) < 3) {
                // only update rarely as we do it every doc
                replicaLocalCheckpoint = randomIntBetween(Math.toIntExact(replicaLocalCheckpoint), Math.toIntExact(primarySeqNo));
            }
            gcpTracker.updateLocalCheckpoint(primary.allocationId().getId(), initialEngine.getPersistedLocalCheckpoint());
            gcpTracker.updateLocalCheckpoint(replica.allocationId().getId(), replicaLocalCheckpoint);
            if (rarely()) {
                localCheckpoint = primarySeqNo;
                maxSeqNo = primarySeqNo;
                initialEngine.flush(true, true);
            }
        }
        logger.info("localcheckpoint {}, global {}", replicaLocalCheckpoint, primarySeqNo);
        globalCheckpoint = gcpTracker.getGlobalCheckpoint();
        assertEquals(primarySeqNo, initialEngine.getSeqNoStats(-1).getMaxSeqNo());
        assertEquals(primarySeqNo, initialEngine.getPersistedLocalCheckpoint());
        assertThat(globalCheckpoint, equalTo(replicaLocalCheckpoint));
        assertThat(Long.parseLong(initialEngine.commitStats().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)), equalTo(localCheckpoint));
        // to guarantee the global checkpoint is written to the translog checkpoint
        initialEngine.getTranslog().sync();
        assertThat(initialEngine.getTranslog().getLastSyncedGlobalCheckpoint(), equalTo(globalCheckpoint));
        assertThat(Long.parseLong(initialEngine.commitStats().getUserData().get(SequenceNumbers.MAX_SEQ_NO)), equalTo(maxSeqNo));
    } finally {
        IOUtils.close(initialEngine);
    }
    try (InternalEngine recoveringEngine = new InternalEngine(initialEngine.config())) {
        recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
        assertEquals(primarySeqNo, recoveringEngine.getSeqNoStats(-1).getMaxSeqNo());
        assertThat(Long.parseLong(recoveringEngine.commitStats().getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)), equalTo(primarySeqNo));
        assertThat(recoveringEngine.getTranslog().getLastSyncedGlobalCheckpoint(), equalTo(globalCheckpoint));
        assertThat(Long.parseLong(recoveringEngine.commitStats().getUserData().get(SequenceNumbers.MAX_SEQ_NO)), // we have assigned sequence numbers to should be in the commit
        equalTo(primarySeqNo));
        assertThat(recoveringEngine.getProcessedLocalCheckpoint(), equalTo(primarySeqNo));
        assertThat(recoveringEngine.getPersistedLocalCheckpoint(), equalTo(primarySeqNo));
        assertThat(recoveringEngine.getSeqNoStats(-1).getMaxSeqNo(), equalTo(primarySeqNo));
        assertThat(generateNewSeqNo(recoveringEngine), equalTo(primarySeqNo + 1));
    }
}

Example 63 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class NoOpEngineTests method testNoopAfterRegularEngine.

public void testNoopAfterRegularEngine() throws IOException {
    int docs = randomIntBetween(1, 10);
    ReplicationTracker tracker = (ReplicationTracker) engine.config().getGlobalCheckpointSupplier();
    ShardRouting routing = TestShardRouting.newShardRouting("test", shardId.id(), "node", null, true, ShardRoutingState.STARTED, allocationId);
    IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(shardId).addShard(routing).build();
    tracker.updateFromMaster(1L, Collections.singleton(allocationId.getId()), table);
    tracker.activatePrimaryMode(SequenceNumbers.NO_OPS_PERFORMED);
    for (int i = 0; i < docs; i++) {
        ParsedDocument doc = testParsedDocument("" + i, null, testDocumentWithTextField(), B_1, null);
        engine.index(indexForDoc(doc));
        tracker.updateLocalCheckpoint(allocationId.getId(), i);
    }
    engine.flush(true, true);
    long localCheckpoint = engine.getPersistedLocalCheckpoint();
    long maxSeqNo = engine.getSeqNoStats(100L).getMaxSeqNo();
    engine.close();
    final NoOpEngine noOpEngine = new NoOpEngine(noOpConfig(INDEX_SETTINGS, store, primaryTranslogDir, tracker));
    assertThat(noOpEngine.getPersistedLocalCheckpoint(), equalTo(localCheckpoint));
    assertThat(noOpEngine.getSeqNoStats(100L).getMaxSeqNo(), equalTo(maxSeqNo));
    try (Engine.IndexCommitRef ref = noOpEngine.acquireLastIndexCommit(false)) {
        try (IndexReader reader = DirectoryReader.open(ref.getIndexCommit())) {
            assertThat(reader.numDocs(), equalTo(docs));
        }
    }
    noOpEngine.close();
}

Example 64 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class IndexLevelReplicationTests method testDocumentFailureReplication.

/**
 * test document failures (failures after seq_no generation) are added as noop operation to the translog
 * for primary and replica shards
 */
public void testDocumentFailureReplication() throws Exception {
    final IOException indexException = new IOException("simulated indexing failure");
    final EngineFactory engineFactory = config -> InternalEngineTests.createInternalEngine((dir, iwc) -> new IndexWriter(dir, iwc) {

        @Override
        public long addDocument(Iterable<? extends IndexableField> doc) throws IOException {
            boolean isTombstone = false;
            for (IndexableField field : doc) {
                if (SeqNoFieldMapper.TOMBSTONE_NAME.equals(field.name())) {
                    isTombstone = true;
                }
            }
            if (isTombstone) {
                // allow to add Noop
                return super.addDocument(doc);
            } else {
                throw indexException;
            }
        }
    }, null, null, config);
    try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetadata(0)) {

        @Override
        protected EngineFactory getEngineFactory(ShardRouting routing) {
            return engineFactory;
        }
    }) {
        // start with the primary only so two first failures are replicated to replicas via recovery from the translog of the primary.
        shards.startPrimary();
        long primaryTerm = shards.getPrimary().getPendingPrimaryTerm();
        List<Translog.Operation> expectedTranslogOps = new ArrayList<>();
        BulkItemResponse indexResp = shards.index(new IndexRequest(index.getName()).id("1").source("{}", XContentType.JSON));
        assertThat(indexResp.isFailed(), equalTo(true));
        assertThat(indexResp.getFailure().getCause(), equalTo(indexException));
        expectedTranslogOps.add(new Translog.NoOp(0, primaryTerm, indexException.toString()));
        try (Translog.Snapshot snapshot = getTranslog(shards.getPrimary()).newSnapshot()) {
            assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
        }
        shards.assertAllEqual(0);
        int nReplica = randomIntBetween(1, 3);
        for (int i = 0; i < nReplica; i++) {
            shards.addReplica();
        }
        shards.startReplicas(nReplica);
        for (IndexShard shard : shards) {
            try (Translog.Snapshot snapshot = getTranslog(shard).newSnapshot()) {
                // we flush at the end of peer recovery
                if (shard.routingEntry().primary() || shard.indexSettings().isSoftDeleteEnabled() == false) {
                    assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
                } else {
                    assertThat(snapshot.totalOperations(), equalTo(0));
                }
            }
            try (Translog.Snapshot snapshot = shard.newChangesSnapshot("test", 0, Long.MAX_VALUE, false)) {
                assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
            }
        }
        // the failure replicated directly from the replication channel.
        indexResp = shards.index(new IndexRequest(index.getName()).id("any").source("{}", XContentType.JSON));
        assertThat(indexResp.getFailure().getCause(), equalTo(indexException));
        Translog.NoOp noop2 = new Translog.NoOp(1, primaryTerm, indexException.toString());
        expectedTranslogOps.add(noop2);
        for (IndexShard shard : shards) {
            try (Translog.Snapshot snapshot = getTranslog(shard).newSnapshot()) {
                if (shard.routingEntry().primary() || shard.indexSettings().isSoftDeleteEnabled() == false) {
                    assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
                } else {
                    assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(Collections.singletonList(noop2)));
                }
            }
            try (Translog.Snapshot snapshot = shard.newChangesSnapshot("test", 0, Long.MAX_VALUE, false)) {
                assertThat(snapshot, SnapshotMatchers.containsOperationsInAnyOrder(expectedTranslogOps));
            }
        }
        shards.assertAllEqual(0);
    }
}

Example 65 with ShardRouting

use of org.opensearch.cluster.routing.ShardRouting in project OpenSearch by opensearch-project.

the class RecoveryDuringReplicationTests method testDoNotWaitForPendingSeqNo.

public void testDoNotWaitForPendingSeqNo() throws Exception {
    IndexMetadata metadata = buildIndexMetadata(1);
    final int pendingDocs = randomIntBetween(1, 5);
    final BlockingEngineFactory primaryEngineFactory = new BlockingEngineFactory();
    try (ReplicationGroup shards = new ReplicationGroup(metadata) {

        @Override
        protected EngineFactory getEngineFactory(ShardRouting routing) {
            if (routing.primary()) {
                return primaryEngineFactory;
            } else {
                return new InternalEngineFactory();
            }
        }
    }) {
        shards.startAll();
        int docs = shards.indexDocs(randomIntBetween(1, 10));
        // simulate a background global checkpoint sync at which point we expect the global checkpoint to advance on the replicas
        shards.syncGlobalCheckpoint();
        IndexShard replica = shards.getReplicas().get(0);
        shards.removeReplica(replica);
        closeShards(replica);
        docs += pendingDocs;
        primaryEngineFactory.latchIndexers(pendingDocs);
        CountDownLatch pendingDocsDone = new CountDownLatch(pendingDocs);
        for (int i = 0; i < pendingDocs; i++) {
            final String id = "pending_" + i;
            threadPool.generic().submit(() -> {
                try {
                    shards.index(new IndexRequest(index.getName()).id(id).source("{}", XContentType.JSON));
                } catch (Exception e) {
                    throw new AssertionError(e);
                } finally {
                    pendingDocsDone.countDown();
                }
            });
        }
        // wait for the pending ops to "hang"
        primaryEngineFactory.awaitIndexersLatch();
        primaryEngineFactory.allowIndexing();
        // index some more
        docs += shards.indexDocs(randomInt(5));
        IndexShard newReplica = shards.addReplicaWithExistingPath(replica.shardPath(), replica.routingEntry().currentNodeId());
        CountDownLatch recoveryStart = new CountDownLatch(1);
        AtomicBoolean recoveryDone = new AtomicBoolean(false);
        final Future<Void> recoveryFuture = shards.asyncRecoverReplica(newReplica, (indexShard, node) -> {
            recoveryStart.countDown();
            return new RecoveryTarget(indexShard, node, recoveryListener) {

                @Override
                public void finalizeRecovery(long globalCheckpoint, long trimAboveSeqNo, ActionListener<Void> listener) {
                    recoveryDone.set(true);
                    super.finalizeRecovery(globalCheckpoint, trimAboveSeqNo, listener);
                }
            };
        });
        recoveryStart.await();
        // index some more
        final int indexedDuringRecovery = shards.indexDocs(randomInt(5));
        docs += indexedDuringRecovery;
        assertBusy(() -> assertTrue("recovery should not wait for on pending docs", recoveryDone.get()));
        primaryEngineFactory.releaseLatchedIndexers();
        pendingDocsDone.await();
        // now recovery can finish
        recoveryFuture.get();
        assertThat(newReplica.recoveryState().getIndex().fileDetails(), empty());
        shards.assertAllEqual(docs);
    } finally {
        primaryEngineFactory.close();
    }
}