Search in sources :

Example 1 with ResyncReplicationRequest

use of org.elasticsearch.action.resync.ResyncReplicationRequest in project crate by crate.

the class PrimaryReplicaSyncerTests method testSyncerSendsOffCorrectDocuments.

@Test
public void testSyncerSendsOffCorrectDocuments() throws Exception {
    IndexShard shard = newStartedShard(true);
    AtomicBoolean syncActionCalled = new AtomicBoolean();
    List<ResyncReplicationRequest> resyncRequests = new ArrayList<>();
    PrimaryReplicaSyncer.SyncAction syncAction = (request, allocationId, primaryTerm, listener) -> {
        logger.info("Sending off {} operations", request.getOperations().length);
        syncActionCalled.set(true);
        resyncRequests.add(request);
        listener.onResponse(new ReplicationResponse());
    };
    PrimaryReplicaSyncer syncer = new PrimaryReplicaSyncer(syncAction);
    syncer.setChunkSize(new ByteSizeValue(randomIntBetween(1, 10)));
    int numDocs = randomInt(10);
    for (int i = 0; i < numDocs; i++) {
        // Index doc but not advance local checkpoint.
        shard.applyIndexOperationOnPrimary(Versions.MATCH_ANY, VersionType.INTERNAL, new SourceToParse(shard.shardId().getIndexName(), Integer.toString(i), new BytesArray("{}"), XContentType.JSON), SequenceNumbers.UNASSIGNED_SEQ_NO, 0, -1L, true);
    }
    long globalCheckPoint = numDocs > 0 ? randomIntBetween(0, numDocs - 1) : 0;
    boolean syncNeeded = numDocs > 0;
    String allocationId = shard.routingEntry().allocationId().getId();
    shard.updateShardState(shard.routingEntry(), shard.getPendingPrimaryTerm(), null, 1000L, Collections.singleton(allocationId), new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build());
    shard.updateLocalCheckpointForShard(allocationId, globalCheckPoint);
    assertEquals(globalCheckPoint, shard.getLastKnownGlobalCheckpoint());
    logger.info("Total ops: {}, global checkpoint: {}", numDocs, globalCheckPoint);
    PlainActionFuture<PrimaryReplicaSyncer.ResyncTask> fut = new PlainActionFuture<>();
    syncer.resync(shard, fut);
    PrimaryReplicaSyncer.ResyncTask resyncTask = fut.get();
    if (syncNeeded) {
        assertTrue("Sync action was not called", syncActionCalled.get());
        ResyncReplicationRequest resyncRequest = resyncRequests.remove(0);
        assertThat(resyncRequest.getTrimAboveSeqNo(), equalTo(numDocs - 1L));
        assertThat("trimAboveSeqNo has to be specified in request #0 only", resyncRequests.stream().mapToLong(ResyncReplicationRequest::getTrimAboveSeqNo).filter(seqNo -> seqNo != SequenceNumbers.UNASSIGNED_SEQ_NO).findFirst().isPresent(), is(false));
        assertThat(resyncRequest.getMaxSeenAutoIdTimestampOnPrimary(), equalTo(shard.getMaxSeenAutoIdTimestamp()));
    }
    if (syncNeeded && globalCheckPoint < numDocs - 1) {
        assertThat(resyncTask.getSkippedOperations(), equalTo(0));
        assertThat(resyncTask.getResyncedOperations(), equalTo(Math.toIntExact(numDocs - 1 - globalCheckPoint)));
        if (shard.indexSettings.isSoftDeleteEnabled()) {
            assertThat(resyncTask.getTotalOperations(), equalTo(Math.toIntExact(numDocs - 1 - globalCheckPoint)));
        } else {
            assertThat(resyncTask.getTotalOperations(), equalTo(numDocs));
        }
    } else {
        assertThat(resyncTask.getSkippedOperations(), equalTo(0));
        assertThat(resyncTask.getResyncedOperations(), equalTo(0));
        assertThat(resyncTask.getTotalOperations(), equalTo(0));
    }
    closeShards(shard);
}
Also used : Arrays(java.util.Arrays) Versions(org.elasticsearch.common.lucene.uid.Versions) IsInstanceOf.instanceOf(org.hamcrest.core.IsInstanceOf.instanceOf) XContentType(org.elasticsearch.common.xcontent.XContentType) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) VersionType(org.elasticsearch.index.VersionType) Matchers.anyString(org.mockito.Matchers.anyString) ArrayList(java.util.ArrayList) BytesArray(org.elasticsearch.common.bytes.BytesArray) Settings(org.elasticsearch.common.settings.Settings) Matchers.eq(org.mockito.Matchers.eq) Matchers.anyLong(org.mockito.Matchers.anyLong) TestTranslog(org.elasticsearch.index.translog.TestTranslog) Mockito.doReturn(org.mockito.Mockito.doReturn) SourceToParse(org.elasticsearch.index.mapper.SourceToParse) ByteSizeValue(org.elasticsearch.common.unit.ByteSizeValue) SequenceNumbers(org.elasticsearch.index.seqno.SequenceNumbers) PlainActionFuture(org.elasticsearch.action.support.PlainActionFuture) IndexShardRoutingTable(org.elasticsearch.cluster.routing.IndexShardRoutingTable) Test(org.junit.Test) Collectors(java.util.stream.Collectors) Engine(org.elasticsearch.index.engine.Engine) CountDownLatch(java.util.concurrent.CountDownLatch) Mockito(org.mockito.Mockito) List(java.util.List) ReplicationResponse(org.elasticsearch.action.support.replication.ReplicationResponse) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Translog(org.elasticsearch.index.translog.Translog) Matchers.is(org.hamcrest.Matchers.is) ResyncReplicationRequest(org.elasticsearch.action.resync.ResyncReplicationRequest) Collections(java.util.Collections) IndexShardRoutingTable(org.elasticsearch.cluster.routing.IndexShardRoutingTable) BytesArray(org.elasticsearch.common.bytes.BytesArray) ArrayList(java.util.ArrayList) ByteSizeValue(org.elasticsearch.common.unit.ByteSizeValue) SourceToParse(org.elasticsearch.index.mapper.SourceToParse) Matchers.anyString(org.mockito.Matchers.anyString) ReplicationResponse(org.elasticsearch.action.support.replication.ReplicationResponse) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) PlainActionFuture(org.elasticsearch.action.support.PlainActionFuture) ResyncReplicationRequest(org.elasticsearch.action.resync.ResyncReplicationRequest) Test(org.junit.Test)

Aggregations

ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1 List (java.util.List)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 Collectors (java.util.stream.Collectors)1 AlreadyClosedException (org.apache.lucene.store.AlreadyClosedException)1 ResyncReplicationRequest (org.elasticsearch.action.resync.ResyncReplicationRequest)1 PlainActionFuture (org.elasticsearch.action.support.PlainActionFuture)1 ReplicationResponse (org.elasticsearch.action.support.replication.ReplicationResponse)1 IndexShardRoutingTable (org.elasticsearch.cluster.routing.IndexShardRoutingTable)1 BytesArray (org.elasticsearch.common.bytes.BytesArray)1 Versions (org.elasticsearch.common.lucene.uid.Versions)1 Settings (org.elasticsearch.common.settings.Settings)1 ByteSizeValue (org.elasticsearch.common.unit.ByteSizeValue)1 XContentType (org.elasticsearch.common.xcontent.XContentType)1 VersionType (org.elasticsearch.index.VersionType)1 Engine (org.elasticsearch.index.engine.Engine)1 SourceToParse (org.elasticsearch.index.mapper.SourceToParse)1