Examples with BackgroundIndexer org.opensearch.test.BackgroundIndexer used on opensource projects

Example 11 with BackgroundIndexer

use of org.opensearch.test.BackgroundIndexer in project OpenSearch by opensearch-project.

the class CloseWhileRelocatingShardsIT method testCloseWhileRelocatingShards.

public void testCloseWhileRelocatingShards() throws Exception {
    final String[] indices = new String[randomIntBetween(3, 5)];
    final Map<String, Long> docsPerIndex = new HashMap<>();
    final Map<String, BackgroundIndexer> indexers = new HashMap<>();
    for (int i = 0; i < indices.length; i++) {
        final String indexName = "index-" + i;
        int nbDocs = 0;
        switch(i) {
            case 0:
                logger.debug("creating empty index {}", indexName);
                createIndex(indexName);
                break;
            case 1:
                nbDocs = scaledRandomIntBetween(1, 100);
                logger.debug("creating index {} with {} documents", indexName, nbDocs);
                createIndex(indexName);
                indexRandom(randomBoolean(), IntStream.range(0, nbDocs).mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(Collectors.toList()));
                break;
            default:
                logger.debug("creating index {} with background indexing", indexName);
                final BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), -1, 1);
                indexers.put(indexName, indexer);
                indexer.setFailureAssertion(t -> assertException(t, indexName));
                waitForDocs(1, indexer);
        }
        docsPerIndex.put(indexName, (long) nbDocs);
        indices[i] = indexName;
    }
    ensureGreen(TimeValue.timeValueSeconds(60L), indices);
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString())));
    final String targetNode = internalCluster().startDataOnlyNode();
    // wait for the cluster-manager to finish processing join.
    ensureClusterSizeConsistency();
    try {
        final ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
        final ClusterState state = clusterService.state();
        final CountDownLatch latch = new CountDownLatch(indices.length);
        final CountDownLatch release = new CountDownLatch(indices.length);
        // relocate one shard for every index to be closed
        final AllocationCommands commands = new AllocationCommands();
        for (final String index : indices) {
            final NumShards numShards = getNumShards(index);
            final int shardId = numShards.numPrimaries == 1 ? 0 : randomIntBetween(0, numShards.numPrimaries - 1);
            final IndexRoutingTable indexRoutingTable = state.routingTable().index(index);
            final ShardRouting primary = indexRoutingTable.shard(shardId).primaryShard();
            assertTrue(primary.started());
            String currentNodeId = primary.currentNodeId();
            if (numShards.numReplicas > 0) {
                final ShardRouting replica = indexRoutingTable.shard(shardId).replicaShards().iterator().next();
                assertTrue(replica.started());
                if (randomBoolean()) {
                    currentNodeId = replica.currentNodeId();
                }
            }
            commands.add(new MoveAllocationCommand(index, shardId, state.nodes().resolveNode(currentNodeId).getName(), targetNode));
        }
        // Build the list of shards for which recoveries will be blocked
        final Set<ShardId> blockedShards = commands.commands().stream().map(c -> (MoveAllocationCommand) c).map(c -> new ShardId(clusterService.state().metadata().index(c.index()).getIndex(), c.shardId())).collect(Collectors.toSet());
        assertThat(blockedShards, hasSize(indices.length));
        final Set<String> acknowledgedCloses = ConcurrentCollections.newConcurrentSet();
        final Set<String> interruptedRecoveries = ConcurrentCollections.newConcurrentSet();
        // Create a SendRequestBehavior that will block outgoing start recovery request
        final StubbableTransport.SendRequestBehavior sendBehavior = (connection, requestId, action, request, options) -> {
            if (PeerRecoverySourceService.Actions.START_RECOVERY.equals(action)) {
                final StartRecoveryRequest startRecoveryRequest = ((StartRecoveryRequest) request);
                if (blockedShards.contains(startRecoveryRequest.shardId())) {
                    logger.debug("blocking recovery of shard {}", startRecoveryRequest.shardId());
                    latch.countDown();
                    try {
                        release.await();
                        logger.debug("releasing recovery of shard {}", startRecoveryRequest.shardId());
                    } catch (final InterruptedException e) {
                        logger.warn(() -> new ParameterizedMessage("exception when releasing recovery of shard {}", startRecoveryRequest.shardId()), e);
                        interruptedRecoveries.add(startRecoveryRequest.shardId().getIndexName());
                        Thread.currentThread().interrupt();
                        return;
                    }
                }
            }
            connection.sendRequest(requestId, action, request, options);
        };
        final MockTransportService targetTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, targetNode);
        for (DiscoveryNode node : state.getNodes()) {
            if (node.isDataNode() && node.getName().equals(targetNode) == false) {
                final TransportService sourceTransportService = internalCluster().getInstance(TransportService.class, node.getName());
                targetTransportService.addSendBehavior(sourceTransportService, sendBehavior);
            }
        }
        assertAcked(client().admin().cluster().reroute(new ClusterRerouteRequest().commands(commands)).get());
        // start index closing threads
        final List<Thread> threads = new ArrayList<>();
        for (final String indexToClose : indices) {
            final Thread thread = new Thread(() -> {
                try {
                    latch.await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                } finally {
                    release.countDown();
                }
                // Closing is not always acknowledged when shards are relocating: this is the case when the target shard is initializing
                // or is catching up operations. In these cases the TransportVerifyShardBeforeCloseAction will detect that the global
                // and max sequence number don't match and will not ack the close.
                AcknowledgedResponse closeResponse = client().admin().indices().prepareClose(indexToClose).get();
                if (closeResponse.isAcknowledged()) {
                    assertTrue("Index closing should not be acknowledged twice", acknowledgedCloses.add(indexToClose));
                }
            });
            threads.add(thread);
            thread.start();
        }
        latch.countDown();
        for (Thread thread : threads) {
            thread.join();
        }
        // stop indexers first without waiting for stop to not redundantly index on some while waiting for another one to stop
        for (BackgroundIndexer indexer : indexers.values()) {
            indexer.stop();
        }
        for (Map.Entry<String, BackgroundIndexer> entry : indexers.entrySet()) {
            final BackgroundIndexer indexer = entry.getValue();
            indexer.awaitStopped();
            final String indexName = entry.getKey();
            docsPerIndex.computeIfPresent(indexName, (key, value) -> value + indexer.totalIndexedDocs());
        }
        for (String index : indices) {
            if (acknowledgedCloses.contains(index)) {
                assertIndexIsClosed(index);
            } else {
                assertIndexIsOpened(index);
            }
        }
        targetTransportService.clearAllRules();
        // If a shard recovery has been interrupted, we expect its index to be closed
        interruptedRecoveries.forEach(CloseIndexIT::assertIndexIsClosed);
        assertThat("Consider that the test failed if no indices were successfully closed", acknowledgedCloses.size(), greaterThan(0));
        assertAcked(client().admin().indices().prepareOpen("index-*"));
        ensureGreen(indices);
        for (String index : acknowledgedCloses) {
            long docsCount = client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get().getHits().getTotalHits().value;
            assertEquals("Expected " + docsPerIndex.get(index) + " docs in index " + index + " but got " + docsCount + " (close acknowledged=" + acknowledgedCloses.contains(index) + ")", (long) docsPerIndex.get(index), docsCount);
        }
    } finally {
        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().putNull(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey())));
    }
}

Example 12 with BackgroundIndexer

use of org.opensearch.test.BackgroundIndexer in project OpenSearch by opensearch-project.

the class RecoveryWhileUnderLoadIT method testRecoverWhileUnderLoadAllocateReplicasTest.

public void testRecoverWhileUnderLoadAllocateReplicasTest() throws Exception {
    logger.info("--> creating test index ...");
    int numberOfShards = numberOfShards();
    assertAcked(prepareCreate("test", 1, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numberOfShards).put(SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC)));
    final int totalNumDocs = scaledRandomIntBetween(200, 10000);
    int waitFor = totalNumDocs / 10;
    int extraDocs = waitFor;
    try (BackgroundIndexer indexer = new BackgroundIndexer("test", "type", client(), extraDocs)) {
        logger.info("--> waiting for {} docs to be indexed ...", waitFor);
        waitForDocs(waitFor, indexer);
        indexer.assertNoFailures();
        logger.info("--> {} docs indexed", waitFor);
        extraDocs = totalNumDocs / 10;
        waitFor += extraDocs;
        indexer.continueIndexing(extraDocs);
        logger.info("--> flushing the index ....");
        // now flush, just to make sure we have some data in the index, not just translog
        client().admin().indices().prepareFlush().execute().actionGet();
        logger.info("--> waiting for {} docs to be indexed ...", waitFor);
        waitForDocs(waitFor, indexer);
        indexer.assertNoFailures();
        logger.info("--> {} docs indexed", waitFor);
        extraDocs = totalNumDocs - waitFor;
        indexer.continueIndexing(extraDocs);
        logger.info("--> allow 2 nodes for index [test] ...");
        // now start another node, while we index
        allowNodes("test", 2);
        logger.info("--> waiting for GREEN health status ...");
        // make sure the cluster state is green, and all has been recovered
        assertNoTimeout(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("5m").setWaitForGreenStatus());
        logger.info("--> waiting for {} docs to be indexed ...", totalNumDocs);
        waitForDocs(totalNumDocs, indexer);
        indexer.assertNoFailures();
        logger.info("--> {} docs indexed", totalNumDocs);
        logger.info("--> marking and waiting for indexing threads to stop ...");
        indexer.stopAndAwaitStopped();
        logger.info("--> indexing threads stopped");
        logger.info("--> refreshing the index");
        refreshAndAssert();
        logger.info("--> verifying indexed content");
        iterateAssertCount(numberOfShards, 10, indexer.getIds());
    }
}

Example 13 with BackgroundIndexer

use of org.opensearch.test.BackgroundIndexer in project OpenSearch by opensearch-project.

the class RecoveryWhileUnderLoadIT method testRecoverWhileRelocating.

public void testRecoverWhileRelocating() throws Exception {
    final int numShards = between(2, 5);
    final int numReplicas = 0;
    logger.info("--> creating test index ...");
    int allowNodes = 2;
    assertAcked(prepareCreate("test", 3, Settings.builder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(SETTING_NUMBER_OF_REPLICAS, numReplicas).put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.ASYNC).put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), randomFrom("100ms", "1s", "5s", "30s", "60s"))));
    final int numDocs = scaledRandomIntBetween(200, 9999);
    try (BackgroundIndexer indexer = new BackgroundIndexer("test", "type", client(), numDocs)) {
        for (int i = 0; i < numDocs; i += scaledRandomIntBetween(100, Math.min(1000, numDocs))) {
            indexer.assertNoFailures();
            logger.info("--> waiting for {} docs to be indexed ...", i);
            waitForDocs(i, indexer);
            logger.info("--> {} docs indexed", i);
            allowNodes = 2 / allowNodes;
            allowNodes("test", allowNodes);
            logger.info("--> waiting for GREEN health status ...");
            ensureGreen(TimeValue.timeValueMinutes(5));
        }
        logger.info("--> marking and waiting for indexing threads to stop ...");
        indexer.stopAndAwaitStopped();
        logger.info("--> indexing threads stopped");
        logger.info("--> bump up number of replicas to 1 and allow all nodes to hold the index");
        allowNodes("test", 3);
        assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put("number_of_replicas", 1)).get());
        ensureGreen(TimeValue.timeValueMinutes(5));
        logger.info("--> refreshing the index");
        refreshAndAssert();
        logger.info("--> verifying indexed content");
        iterateAssertCount(numShards, 10, indexer.getIds());
    }
}

Example 14 with BackgroundIndexer

use of org.opensearch.test.BackgroundIndexer in project OpenSearch by opensearch-project.

the class RelocationIT method testRelocationWhileIndexingRandom.

@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/2063")
public void testRelocationWhileIndexingRandom() throws Exception {
    int numberOfRelocations = scaledRandomIntBetween(1, rarely() ? 10 : 4);
    int numberOfReplicas = randomBoolean() ? 0 : 1;
    int numberOfNodes = numberOfReplicas == 0 ? 2 : 3;
    logger.info("testRelocationWhileIndexingRandom(numRelocations={}, numberOfReplicas={}, numberOfNodes={})", numberOfRelocations, numberOfReplicas, numberOfNodes);
    String[] nodes = new String[numberOfNodes];
    logger.info("--> starting [node1] ...");
    nodes[0] = internalCluster().startNode();
    logger.info("--> creating test index ...");
    prepareCreate("test", Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", numberOfReplicas)).get();
    for (int i = 2; i <= numberOfNodes; i++) {
        logger.info("--> starting [node{}] ...", i);
        nodes[i - 1] = internalCluster().startNode();
        if (i != numberOfNodes) {
            ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes(Integer.toString(i)).setWaitForGreenStatus().execute().actionGet();
            assertThat(healthResponse.isTimedOut(), equalTo(false));
        }
    }
    int numDocs = scaledRandomIntBetween(200, 2500);
    try (BackgroundIndexer indexer = new BackgroundIndexer("test", MapperService.SINGLE_MAPPING_NAME, client(), numDocs)) {
        logger.info("--> waiting for {} docs to be indexed ...", numDocs);
        waitForDocs(numDocs, indexer);
        logger.info("--> {} docs indexed", numDocs);
        logger.info("--> starting relocations...");
        // if we have replicas shift those
        int nodeShiftBased = numberOfReplicas;
        for (int i = 0; i < numberOfRelocations; i++) {
            int fromNode = (i % 2);
            int toNode = fromNode == 0 ? 1 : 0;
            fromNode += nodeShiftBased;
            toNode += nodeShiftBased;
            numDocs = scaledRandomIntBetween(200, 1000);
            logger.debug("--> Allow indexer to index [{}] documents", numDocs);
            indexer.continueIndexing(numDocs);
            logger.info("--> START relocate the shard from {} to {}", nodes[fromNode], nodes[toNode]);
            client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, nodes[fromNode], nodes[toNode])).get();
            if (rarely()) {
                logger.debug("--> flushing");
                client().admin().indices().prepareFlush().get();
            }
            ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNoRelocatingShards(true).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
            assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
            indexer.pauseIndexing();
            logger.info("--> DONE relocate the shard from {} to {}", fromNode, toNode);
        }
        logger.info("--> done relocations");
        logger.info("--> waiting for indexing threads to stop ...");
        indexer.stopAndAwaitStopped();
        logger.info("--> indexing threads stopped");
        logger.info("--> refreshing the index");
        client().admin().indices().prepareRefresh("test").execute().actionGet();
        logger.info("--> searching the index");
        boolean ranOnce = false;
        for (int i = 0; i < 10; i++) {
            logger.info("--> START search test round {}", i + 1);
            SearchHits hits = client().prepareSearch("test").setQuery(matchAllQuery()).setSize((int) indexer.totalIndexedDocs()).storedFields().execute().actionGet().getHits();
            ranOnce = true;
            if (hits.getTotalHits().value != indexer.totalIndexedDocs()) {
                int[] hitIds = new int[(int) indexer.totalIndexedDocs()];
                for (int hit = 0; hit < indexer.totalIndexedDocs(); hit++) {
                    hitIds[hit] = hit + 1;
                }
                Set<Integer> set = Arrays.stream(hitIds).boxed().collect(Collectors.toSet());
                for (SearchHit hit : hits.getHits()) {
                    int id = Integer.parseInt(hit.getId());
                    if (set.remove(id) == false) {
                        logger.error("Extra id [{}]", id);
                    }
                }
                set.forEach(value -> logger.error("Missing id [{}]", value));
            }
            assertThat(hits.getTotalHits().value, equalTo(indexer.totalIndexedDocs()));
            logger.info("--> DONE search test round {}", i + 1);
        }
        if (ranOnce == false) {
            fail();
        }
    }
}