Examples with PlainActionFuture org.elasticsearch.action.support.PlainActionFuture used on opensource projects

Example 36 with PlainActionFuture

use of org.elasticsearch.action.support.PlainActionFuture in project crate by crate.

the class RecoverySourceHandlerTests method testSendFileChunksConcurrently.

@Test
public void testSendFileChunksConcurrently() throws Exception {
    final IndexShard shard = mock(IndexShard.class);
    when(shard.state()).thenReturn(IndexShardState.STARTED);
    final List<FileChunkResponse> unrepliedChunks = new CopyOnWriteArrayList<>();
    final AtomicInteger sentChunks = new AtomicInteger();
    final TestRecoveryTargetHandler recoveryTarget = new TestRecoveryTargetHandler() {

        final AtomicLong chunkNumberGenerator = new AtomicLong();

        @Override
        public void writeFileChunk(StoreFileMetadata md, long position, BytesReference content, boolean lastChunk, int totalTranslogOps, ActionListener<Void> listener) {
            final long chunkNumber = chunkNumberGenerator.getAndIncrement();
            logger.info("--> write chunk name={} seq={}, position={}", md.name(), chunkNumber, position);
            unrepliedChunks.add(new FileChunkResponse(chunkNumber, listener));
            sentChunks.incrementAndGet();
        }
    };
    final int maxConcurrentChunks = between(1, 8);
    final int chunkSize = between(1, 32);
    final RecoverySourceHandler handler = new RecoverySourceHandler(shard, recoveryTarget, threadPool, getStartRecoveryRequest(), chunkSize, maxConcurrentChunks, between(1, 10));
    Store store = newStore(createTempDir(), false);
    List<StoreFileMetadata> files = generateFiles(store, between(1, 10), () -> between(1, chunkSize * 20));
    int totalChunks = files.stream().mapToInt(md -> ((int) md.length() + chunkSize - 1) / chunkSize).sum();
    PlainActionFuture<Void> sendFilesFuture = new PlainActionFuture<>();
    handler.sendFiles(store, files.toArray(new StoreFileMetadata[0]), () -> 0, sendFilesFuture);
    assertBusy(() -> {
        assertThat(sentChunks.get(), equalTo(Math.min(totalChunks, maxConcurrentChunks)));
        assertThat(unrepliedChunks, hasSize(sentChunks.get()));
    });
    List<FileChunkResponse> ackedChunks = new ArrayList<>();
    while (sentChunks.get() < totalChunks || unrepliedChunks.isEmpty() == false) {
        List<FileChunkResponse> chunksToAck = randomSubsetOf(between(1, unrepliedChunks.size()), unrepliedChunks);
        unrepliedChunks.removeAll(chunksToAck);
        ackedChunks.addAll(chunksToAck);
        ackedChunks.sort(Comparator.comparing(c -> c.chunkNumber));
        int checkpoint = -1;
        for (int i = 0; i < ackedChunks.size(); i++) {
            if (i != ackedChunks.get(i).chunkNumber) {
                break;
            } else {
                checkpoint = i;
            }
        }
        int chunksToSend = Math.min(// limited by the remaining chunks
        totalChunks - sentChunks.get(), // limited by the buffering chunks
        maxConcurrentChunks - (sentChunks.get() - 1 - checkpoint));
        int expectedSentChunks = sentChunks.get() + chunksToSend;
        int expectedUnrepliedChunks = unrepliedChunks.size() + chunksToSend;
        chunksToAck.forEach(c -> c.listener.onResponse(null));
        assertBusy(() -> {
            assertThat(sentChunks.get(), equalTo(expectedSentChunks));
            assertThat(unrepliedChunks, hasSize(expectedUnrepliedChunks));
        });
    }
    sendFilesFuture.actionGet();
    store.close();
}

Example 37 with PlainActionFuture

use of org.elasticsearch.action.support.PlainActionFuture in project crate by crate.

the class RecoverySourceHandlerTests method testSendFiles.

@Test
public void testSendFiles() throws Throwable {
    final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service);
    final StartRecoveryRequest request = getStartRecoveryRequest();
    Store store = newStore(createTempDir());
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata(null);
    List<StoreFileMetadata> metas = new ArrayList<>();
    for (StoreFileMetadata md : metadata) {
        metas.add(md);
    }
    Store targetStore = newStore(createTempDir());
    MultiFileWriter multiFileWriter = new MultiFileWriter(targetStore, mock(RecoveryState.Index.class), "", logger, () -> {
    });
    RecoveryTargetHandler target = new TestRecoveryTargetHandler() {

        @Override
        public void writeFileChunk(StoreFileMetadata md, long position, BytesReference content, boolean lastChunk, int totalTranslogOps, ActionListener<Void> listener) {
            ActionListener.completeWith(listener, () -> {
                multiFileWriter.writeFileChunk(md, position, content, lastChunk);
                return null;
            });
        }
    };
    RecoverySourceHandler handler = new RecoverySourceHandler(null, new AsyncRecoveryTarget(target, recoveryExecutor), threadPool, request, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), between(1, 5), between(1, 5));
    PlainActionFuture<Void> sendFilesFuture = new PlainActionFuture<>();
    handler.sendFiles(store, metas.toArray(new StoreFileMetadata[0]), () -> 0, sendFilesFuture);
    sendFilesFuture.actionGet(5, TimeUnit.SECONDS);
    Store.MetadataSnapshot targetStoreMetadata = targetStore.getMetadata(null);
    Store.RecoveryDiff recoveryDiff = targetStoreMetadata.recoveryDiff(metadata);
    assertEquals(metas.size(), recoveryDiff.identical.size());
    assertEquals(0, recoveryDiff.different.size());
    assertEquals(0, recoveryDiff.missing.size());
    IndexReader reader = DirectoryReader.open(targetStore.directory());
    assertEquals(numDocs, reader.maxDoc());
    IOUtils.close(reader, store, multiFileWriter, targetStore);
}

Example 38 with PlainActionFuture

use of org.elasticsearch.action.support.PlainActionFuture in project crate by crate.

the class RecoverySourceHandlerTests method testThrowExceptionOnPrimaryRelocatedBeforePhase1Started.

@Test
public void testThrowExceptionOnPrimaryRelocatedBeforePhase1Started() throws IOException {
    final RecoverySettings recoverySettings = new RecoverySettings(Settings.EMPTY, service);
    final StartRecoveryRequest request = getStartRecoveryRequest();
    final IndexShard shard = mock(IndexShard.class);
    when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class));
    when(shard.isRelocatedPrimary()).thenReturn(true);
    when(shard.acquireSafeIndexCommit()).thenReturn(mock(Engine.IndexCommitRef.class));
    doAnswer(invocation -> {
        ((ActionListener<Releasable>) invocation.getArguments()[0]).onResponse(() -> {
        });
        return null;
    }).when(shard).acquirePrimaryOperationPermit(any(), anyString(), anyObject());
    final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("test").settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, between(0, 5)).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, between(1, 5)).put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), randomBoolean()).put(IndexMetadata.SETTING_VERSION_CREATED, VersionUtils.randomVersion(random())).put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID(random())));
    if (randomBoolean()) {
        indexMetadata.state(IndexMetadata.State.CLOSE);
    }
    when(shard.indexSettings()).thenReturn(new IndexSettings(indexMetadata.build(), Settings.EMPTY));
    final AtomicBoolean phase1Called = new AtomicBoolean();
    final AtomicBoolean prepareTargetForTranslogCalled = new AtomicBoolean();
    final AtomicBoolean phase2Called = new AtomicBoolean();
    final RecoverySourceHandler handler = new RecoverySourceHandler(shard, mock(RecoveryTargetHandler.class), threadPool, request, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), between(1, 8), between(1, 8)) {

        @Override
        void phase1(IndexCommit snapshot, long startingSeqNo, IntSupplier translogOps, ActionListener<SendFileResult> listener) {
            phase1Called.set(true);
            super.phase1(snapshot, startingSeqNo, translogOps, listener);
        }

        @Override
        void prepareTargetForTranslog(int totalTranslogOps, ActionListener<TimeValue> listener) {
            prepareTargetForTranslogCalled.set(true);
            super.prepareTargetForTranslog(totalTranslogOps, listener);
        }

        @Override
        void phase2(long startingSeqNo, long endingSeqNo, Translog.Snapshot snapshot, long maxSeenAutoIdTimestamp, long maxSeqNoOfUpdatesOrDeletes, RetentionLeases retentionLeases, long mappingVersion, ActionListener<SendSnapshotResult> listener) throws IOException {
            phase2Called.set(true);
            super.phase2(startingSeqNo, endingSeqNo, snapshot, maxSeenAutoIdTimestamp, maxSeqNoOfUpdatesOrDeletes, retentionLeases, mappingVersion, listener);
        }
    };
    PlainActionFuture<RecoveryResponse> future = new PlainActionFuture<>();
    expectThrows(IndexShardRelocatedException.class, () -> {
        handler.recoverToTarget(future);
        future.actionGet();
    });
    assertFalse(phase1Called.get());
    assertFalse(prepareTargetForTranslogCalled.get());
    assertFalse(phase2Called.get());
}

Example 39 with PlainActionFuture

use of org.elasticsearch.action.support.PlainActionFuture in project crate by crate.

the class PeerRecoveryTargetServiceTests method testWriteFileChunksConcurrently.

@Test
public void testWriteFileChunksConcurrently() throws Exception {
    IndexShard sourceShard = newStartedShard(true);
    int numDocs = between(20, 100);
    for (int i = 0; i < numDocs; i++) {
        indexDoc(sourceShard, "_doc", Integer.toString(i));
    }
    sourceShard.flush(new FlushRequest());
    Store.MetadataSnapshot sourceSnapshot = sourceShard.store().getMetadata(null);
    List<StoreFileMetadata> mdFiles = new ArrayList<>();
    for (StoreFileMetadata md : sourceSnapshot) {
        mdFiles.add(md);
    }
    final IndexShard targetShard = newShard(false);
    final DiscoveryNode pNode = getFakeDiscoNode(sourceShard.routingEntry().currentNodeId());
    final DiscoveryNode rNode = getFakeDiscoNode(targetShard.routingEntry().currentNodeId());
    targetShard.markAsRecovering("test-peer-recovery", new RecoveryState(targetShard.routingEntry(), rNode, pNode));
    final RecoveryTarget recoveryTarget = new RecoveryTarget(targetShard, null, null);
    final PlainActionFuture<Void> receiveFileInfoFuture = new PlainActionFuture<>();
    recoveryTarget.receiveFileInfo(mdFiles.stream().map(StoreFileMetadata::name).collect(Collectors.toList()), mdFiles.stream().map(StoreFileMetadata::length).collect(Collectors.toList()), Collections.emptyList(), Collections.emptyList(), 0, receiveFileInfoFuture);
    receiveFileInfoFuture.actionGet(5, TimeUnit.SECONDS);
    List<RecoveryFileChunkRequest> requests = new ArrayList<>();
    for (StoreFileMetadata md : mdFiles) {
        try (IndexInput in = sourceShard.store().directory().openInput(md.name(), IOContext.READONCE)) {
            int pos = 0;
            while (pos < md.length()) {
                int length = between(1, Math.toIntExact(md.length() - pos));
                byte[] buffer = new byte[length];
                in.readBytes(buffer, 0, length);
                requests.add(new RecoveryFileChunkRequest(0, sourceShard.shardId(), md, pos, new BytesArray(buffer), pos + length == md.length(), 1, 1));
                pos += length;
            }
        }
    }
    Randomness.shuffle(requests);
    BlockingQueue<RecoveryFileChunkRequest> queue = new ArrayBlockingQueue<>(requests.size());
    queue.addAll(requests);
    Thread[] senders = new Thread[between(1, 4)];
    CyclicBarrier barrier = new CyclicBarrier(senders.length);
    for (int i = 0; i < senders.length; i++) {
        senders[i] = new Thread(() -> {
            try {
                barrier.await();
                RecoveryFileChunkRequest r;
                while ((r = queue.poll()) != null) {
                    recoveryTarget.writeFileChunk(r.metadata(), r.position(), r.content(), r.lastChunk(), r.totalTranslogOps(), ActionListener.wrap(ignored -> {
                    }, e -> {
                        throw new AssertionError(e);
                    }));
                }
            } catch (Exception e) {
                throw new AssertionError(e);
            }
        });
        senders[i].start();
    }
    for (Thread sender : senders) {
        sender.join();
    }
    PlainActionFuture<Void> cleanFilesFuture = new PlainActionFuture<>();
    recoveryTarget.cleanFiles(0, Long.parseLong(sourceSnapshot.getCommitUserData().get(SequenceNumbers.MAX_SEQ_NO)), sourceSnapshot, cleanFilesFuture);
    cleanFilesFuture.actionGet();
    recoveryTarget.decRef();
    Store.MetadataSnapshot targetSnapshot = targetShard.snapshotStoreMetadata();
    Store.RecoveryDiff diff = sourceSnapshot.recoveryDiff(targetSnapshot);
    assertThat(diff.different, empty());
    closeShards(sourceShard, targetShard);
}

Example 40 with PlainActionFuture

use of org.elasticsearch.action.support.PlainActionFuture in project crate by crate.

the class PrimaryReplicaSyncerTests method testDoNotSendOperationsWithoutSequenceNumber.

@Test
public void testDoNotSendOperationsWithoutSequenceNumber() throws Exception {
    IndexShard shard = Mockito.spy(newStartedShard(true));
    Mockito.when(shard.getLastKnownGlobalCheckpoint()).thenReturn(SequenceNumbers.UNASSIGNED_SEQ_NO);
    int numOps = between(0, 20);
    List<Translog.Operation> operations = new ArrayList<>();
    for (int i = 0; i < numOps; i++) {
        operations.add(new Translog.Index(Integer.toString(i), randomBoolean() ? SequenceNumbers.UNASSIGNED_SEQ_NO : i, primaryTerm, new byte[] { 1 }));
    }
    Engine.HistorySource source = shard.indexSettings.isSoftDeleteEnabled() ? Engine.HistorySource.INDEX : Engine.HistorySource.TRANSLOG;
    doReturn(TestTranslog.newSnapshotFromOperations(operations)).when(shard).getHistoryOperations(anyString(), eq(source), anyLong());
    List<Translog.Operation> sentOperations = new ArrayList<>();
    PrimaryReplicaSyncer.SyncAction syncAction = (request, allocationId, primaryTerm, listener) -> {
        sentOperations.addAll(Arrays.asList(request.getOperations()));
        listener.onResponse(new ReplicationResponse());
    };
    PrimaryReplicaSyncer syncer = new PrimaryReplicaSyncer(syncAction);
    syncer.setChunkSize(new ByteSizeValue(randomIntBetween(1, 10)));
    PlainActionFuture<PrimaryReplicaSyncer.ResyncTask> fut = new PlainActionFuture<>();
    syncer.resync(shard, fut);
    fut.actionGet();
    assertThat(sentOperations, equalTo(operations.stream().filter(op -> op.seqNo() >= 0).collect(Collectors.toList())));
    closeShards(shard);
}