Examples with IndicesService org.opensearch.indices.IndicesService used on opensource projects

Example 6 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class ShardStateIT method testPrimaryFailureIncreasesTerm.

public void testPrimaryFailureIncreasesTerm() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(2);
    prepareCreate("test").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)).get();
    ensureGreen();
    assertPrimaryTerms(1, 1);
    logger.info("--> disabling allocation to capture shard failure");
    disableAllocation("test");
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    final int shard = randomBoolean() ? 0 : 1;
    final String nodeId = state.routingTable().index("test").shard(shard).primaryShard().currentNodeId();
    final String node = state.nodes().get(nodeId).getName();
    logger.info("--> failing primary of [{}] on node [{}]", shard, node);
    IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
    indicesService.indexService(resolveIndex("test")).getShard(shard).failShard("simulated test failure", null);
    logger.info("--> waiting for a yellow index");
    // we can't use ensureYellow since that one is just as happy with a GREEN status.
    assertBusy(() -> assertThat(client().admin().cluster().prepareHealth("test").get().getStatus(), equalTo(ClusterHealthStatus.YELLOW)));
    final long term0 = shard == 0 ? 2 : 1;
    final long term1 = shard == 1 ? 2 : 1;
    assertPrimaryTerms(term0, term1);
    logger.info("--> enabling allocation");
    enableAllocation("test");
    ensureGreen();
    assertPrimaryTerms(term0, term1);
}

Example 7 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class ClusterInfoServiceIT method testClusterInfoServiceCollectsInformation.

public void testClusterInfoServiceCollectsInformation() {
    internalCluster().startNodes(2);
    final String indexName = randomBoolean() ? randomAlphaOfLength(5).toLowerCase(Locale.ROOT) : TEST_SYSTEM_INDEX_NAME;
    assertAcked(prepareCreate(indexName).setSettings(Settings.builder().put(Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), 0).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE).put(IndexMetadata.SETTING_INDEX_HIDDEN, randomBoolean()).build()));
    if (randomBoolean()) {
        assertAcked(client().admin().indices().prepareClose(indexName));
    }
    ensureGreen(indexName);
    InternalTestCluster internalTestCluster = internalCluster();
    // Get the cluster info service on the cluster-manager node
    final InternalClusterInfoService infoService = (InternalClusterInfoService) internalTestCluster.getInstance(ClusterInfoService.class, internalTestCluster.getMasterName());
    infoService.setUpdateFrequency(TimeValue.timeValueMillis(200));
    ClusterInfo info = infoService.refresh();
    assertNotNull("info should not be null", info);
    ImmutableOpenMap<String, DiskUsage> leastUsages = info.getNodeLeastAvailableDiskUsages();
    ImmutableOpenMap<String, DiskUsage> mostUsages = info.getNodeMostAvailableDiskUsages();
    ImmutableOpenMap<String, Long> shardSizes = info.shardSizes;
    assertNotNull(leastUsages);
    assertNotNull(shardSizes);
    assertThat("some usages are populated", leastUsages.values().size(), Matchers.equalTo(2));
    assertThat("some shard sizes are populated", shardSizes.values().size(), greaterThan(0));
    for (ObjectCursor<DiskUsage> usage : leastUsages.values()) {
        logger.info("--> usage: {}", usage.value);
        assertThat("usage has be retrieved", usage.value.getFreeBytes(), greaterThan(0L));
    }
    for (ObjectCursor<DiskUsage> usage : mostUsages.values()) {
        logger.info("--> usage: {}", usage.value);
        assertThat("usage has be retrieved", usage.value.getFreeBytes(), greaterThan(0L));
    }
    for (ObjectCursor<Long> size : shardSizes.values()) {
        logger.info("--> shard size: {}", size.value);
        assertThat("shard size is greater than 0", size.value, greaterThanOrEqualTo(0L));
    }
    ClusterService clusterService = internalTestCluster.getInstance(ClusterService.class, internalTestCluster.getMasterName());
    ClusterState state = clusterService.state();
    for (ShardRouting shard : state.routingTable().allShards()) {
        String dataPath = info.getDataPath(shard);
        assertNotNull(dataPath);
        String nodeId = shard.currentNodeId();
        DiscoveryNode discoveryNode = state.getNodes().get(nodeId);
        IndicesService indicesService = internalTestCluster.getInstance(IndicesService.class, discoveryNode.getName());
        IndexService indexService = indicesService.indexService(shard.index());
        IndexShard indexShard = indexService.getShardOrNull(shard.id());
        assertEquals(indexShard.shardPath().getRootDataPath().toString(), dataPath);
        assertTrue(info.getReservedSpace(nodeId, dataPath).containsShardId(shard.shardId()));
    }
}

Example 8 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class SharedClusterSnapshotRestoreIT method testSnapshotCanceledOnRemovedShard.

/**
 * This test ensures that when a shard is removed from a node (perhaps due to the node
 * leaving the cluster, then returning), all snapshotting of that shard is aborted, so
 * all Store references held onto by the snapshot are released.
 *
 * See https://github.com/elastic/elasticsearch/issues/20876
 */
public void testSnapshotCanceledOnRemovedShard() throws Exception {
    final int numPrimaries = 1;
    final int numReplicas = 1;
    final String repo = "test-repo";
    final String index = "test-idx";
    final String snapshot = "test-snap";
    assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", numPrimaries).put("number_of_replicas", numReplicas)));
    indexRandomDocs(index, 100);
    createRepository(repo, "mock", Settings.builder().put("location", randomRepoPath()).put("random", randomAlphaOfLength(10)).put("wait_after_unblock", 200));
    String blockedNode = blockNodeWithIndex(repo, index);
    logger.info("--> snapshot");
    clusterAdmin().prepareCreateSnapshot(repo, snapshot).setWaitForCompletion(false).execute();
    logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
    waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10));
    logger.info("--> removing primary shard that is being snapshotted");
    ClusterState clusterState = internalCluster().clusterService(internalCluster().getMasterName()).state();
    IndexRoutingTable indexRoutingTable = clusterState.getRoutingTable().index(index);
    String nodeWithPrimary = clusterState.nodes().get(indexRoutingTable.shard(0).primaryShard().currentNodeId()).getName();
    assertNotNull("should be at least one node with a primary shard", nodeWithPrimary);
    IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeWithPrimary);
    IndexService indexService = indicesService.indexService(resolveIndex(index));
    indexService.removeShard(0, "simulate node removal");
    logger.info("--> unblocking blocked node [{}]", blockedNode);
    unblockNode(repo, blockedNode);
    logger.info("--> ensuring snapshot is aborted and the aborted shard was marked as failed");
    SnapshotInfo snapshotInfo = waitForCompletion(repo, snapshot, TimeValue.timeValueSeconds(60));
    assertEquals(1, snapshotInfo.shardFailures().size());
    assertEquals(0, snapshotInfo.shardFailures().get(0).shardId());
    assertThat(snapshotInfo.shardFailures().get(0).reason(), is("aborted"));
}

Example 9 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class IndexRecoveryIT method testRerouteRecovery.

public void testRerouteRecovery() throws Exception {
    logger.info("--> start node A");
    final String nodeA = internalCluster().startNode();
    logger.info("--> create index on node: {}", nodeA);
    ByteSizeValue shardSize = createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT).getShards()[0].getStats().getStore().size();
    logger.info("--> start node B");
    final String nodeB = internalCluster().startNode();
    ensureGreen();
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move shard from: {} to: {}", nodeA, nodeB);
    client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeB)).execute().actionGet().getState();
    logger.info("--> waiting for recovery to start both on source and target");
    final Index index = resolveIndex(INDEX_NAME);
    assertBusy(() -> {
        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA);
        assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(), equalTo(1));
        indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
        assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(), equalTo(1));
    });
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    List<RecoveryState> nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, RecoverySource.EmptyStoreRecoverySource.INSTANCE, true, Stage.DONE, null, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    logger.info("--> request node recovery stats");
    NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    long nodeAThrottling = Long.MAX_VALUE;
    long nodeBThrottling = Long.MAX_VALUE;
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A should have ongoing recovery as source", recoveryStats.currentAsSource(), equalTo(1));
            assertThat("node A should not have ongoing recovery as target", recoveryStats.currentAsTarget(), equalTo(0));
            nodeAThrottling = recoveryStats.throttleTime().millis();
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B should not have ongoing recovery as source", recoveryStats.currentAsSource(), equalTo(0));
            assertThat("node B should have ongoing recovery as target", recoveryStats.currentAsTarget(), equalTo(1));
            nodeBThrottling = recoveryStats.throttleTime().millis();
        }
    }
    logger.info("--> checking throttling increases");
    final long finalNodeAThrottling = nodeAThrottling;
    final long finalNodeBThrottling = nodeBThrottling;
    assertBusy(() -> {
        NodesStatsResponse statsResponse1 = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
        assertThat(statsResponse1.getNodes(), hasSize(2));
        for (NodeStats nodeStats : statsResponse1.getNodes()) {
            final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
            if (nodeStats.getNode().getName().equals(nodeA)) {
                assertThat("node A throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeAThrottling));
            }
            if (nodeStats.getNode().getName().equals(nodeB)) {
                assertThat("node B throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeBThrottling));
            }
        }
    });
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    // wait for it to be finished
    ensureGreen();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(1));
    assertRecoveryState(recoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(recoveryStates.get(0).getIndex());
    Consumer<String> assertNodeHasThrottleTimeAndNoRecoveries = nodeName -> {
        NodesStatsResponse nodesStatsResponse = client().admin().cluster().prepareNodesStats().setNodesIds(nodeName).clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
        assertThat(nodesStatsResponse.getNodes(), hasSize(1));
        NodeStats nodeStats = nodesStatsResponse.getNodes().get(0);
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        assertThat(recoveryStats.currentAsSource(), equalTo(0));
        assertThat(recoveryStats.currentAsTarget(), equalTo(0));
        assertThat(nodeName + " throttling should be >0", recoveryStats.throttleTime().millis(), greaterThan(0L));
    };
    // we have to use assertBusy as recovery counters are decremented only when the last reference to the RecoveryTarget
    // is decremented, which may happen after the recovery was done.
    assertBusy(() -> assertNodeHasThrottleTimeAndNoRecoveries.accept(nodeA));
    assertBusy(() -> assertNodeHasThrottleTimeAndNoRecoveries.accept(nodeB));
    logger.info("--> bump replica count");
    client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put("number_of_replicas", 1)).execute().actionGet();
    ensureGreen();
    assertBusy(() -> assertNodeHasThrottleTimeAndNoRecoveries.accept(nodeA));
    assertBusy(() -> assertNodeHasThrottleTimeAndNoRecoveries.accept(nodeB));
    logger.info("--> start node C");
    String nodeC = internalCluster().startNode();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").get().isTimedOut());
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move replica shard from: {} to: {}", nodeA, nodeC);
    client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeC)).execute().actionGet().getState();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    if (randomBoolean()) {
        // shutdown node with relocation source of replica shard and check if recovery continues
        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
        ensureStableCluster(2);
        response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
        recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
        nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
        assertThat(nodeARecoveryStates.size(), equalTo(0));
        nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
        assertThat(nodeBRecoveryStates.size(), equalTo(1));
        nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
        assertThat(nodeCRecoveryStates.size(), equalTo(1));
        assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
        validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
        assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
        validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    }
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    ensureGreen();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(0));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
}

Example 10 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class IndexShardIT method testStressMaybeFlushOrRollTranslogGeneration.

public void testStressMaybeFlushOrRollTranslogGeneration() throws Exception {
    createIndex("test");
    ensureGreen();
    IndicesService indicesService = getInstanceFromNode(IndicesService.class);
    IndexService test = indicesService.indexService(resolveIndex("test"));
    final IndexShard shard = test.getShardOrNull(0);
    assertFalse(shard.shouldPeriodicallyFlush());
    final boolean flush = randomBoolean();
    final Settings settings;
    if (flush) {
        // size of the operation plus the overhead of one generation.
        settings = Settings.builder().put("index.translog.flush_threshold_size", "125b").build();
    } else {
        // size of the operation plus header and footer
        settings = Settings.builder().put("index.translog.generation_threshold_size", "117b").build();
    }
    client().admin().indices().prepareUpdateSettings("test").setSettings(settings).get();
    client().prepareIndex("test").setId("0").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
    assertFalse(shard.shouldPeriodicallyFlush());
    final AtomicBoolean running = new AtomicBoolean(true);
    final int numThreads = randomIntBetween(2, 4);
    final Thread[] threads = new Thread[numThreads];
    final CyclicBarrier barrier = new CyclicBarrier(numThreads + 1);
    for (int i = 0; i < threads.length; i++) {
        threads[i] = new Thread(() -> {
            try {
                barrier.await();
            } catch (final InterruptedException | BrokenBarrierException e) {
                throw new RuntimeException(e);
            }
            while (running.get()) {
                shard.afterWriteOperation();
            }
        });
        threads[i].start();
    }
    barrier.await();
    final CheckedRunnable<Exception> check;
    if (flush) {
        final FlushStats initialStats = shard.flushStats();
        client().prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).get();
        check = () -> {
            assertFalse(shard.shouldPeriodicallyFlush());
            final FlushStats currentStats = shard.flushStats();
            String msg = String.format(Locale.ROOT, "flush stats: total=[%d vs %d], periodic=[%d vs %d]", initialStats.getTotal(), currentStats.getTotal(), initialStats.getPeriodic(), currentStats.getPeriodic());
            assertThat(msg, currentStats.getPeriodic(), either(equalTo(initialStats.getPeriodic() + 1)).or(equalTo(initialStats.getPeriodic() + 2)));
            assertThat(msg, currentStats.getTotal(), either(equalTo(initialStats.getTotal() + 1)).or(equalTo(initialStats.getTotal() + 2)));
        };
    } else {
        final long generation = getTranslog(shard).currentFileGeneration();
        client().prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).get();
        check = () -> {
            assertFalse(shard.shouldRollTranslogGeneration());
            assertEquals(generation + 1, getTranslog(shard).currentFileGeneration());
        };
    }
    assertBusy(check);
    running.set(false);
    for (int i = 0; i < threads.length; i++) {
        threads[i].join();
    }
    check.run();
}