Examples with IndexService org.opensearch.index.IndexService used on opensource projects

Example 11 with IndexService

use of org.opensearch.index.IndexService in project OpenSearch by opensearch-project.

the class ShrinkIndexIT method testShrinkCommitsMergeOnIdle.

public void testShrinkCommitsMergeOnIdle() throws Exception {
    prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("index.number_of_replicas", 0).put("number_of_shards", 5)).get();
    for (int i = 0; i < 30; i++) {
        client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
    }
    client().admin().indices().prepareFlush("source").get();
    ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes();
    DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
    // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
    // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
    // to the require._name below.
    ensureGreen();
    // relocate all shards to one node such that we can merge it.
    client().admin().indices().prepareUpdateSettings("source").setSettings(Settings.builder().put("index.routing.allocation.require._name", discoveryNodes[0].getName()).put("index.blocks.write", true)).get();
    ensureGreen();
    IndicesSegmentResponse sourceStats = client().admin().indices().prepareSegments("source").get();
    // disable rebalancing to be able to capture the right stats. balancing can move the target primary
    // making it hard to pin point the source shards.
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")).get();
    // now merge source into a single shard index
    assertAcked(client().admin().indices().prepareResizeIndex("source", "target").setSettings(Settings.builder().put("index.number_of_replicas", 0).build()).get());
    ensureGreen();
    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
    IndexMetadata target = clusterStateResponse.getState().getMetadata().index("target");
    client().admin().indices().prepareForceMerge("target").setMaxNumSegments(1).setFlush(false).get();
    IndicesSegmentResponse targetSegStats = client().admin().indices().prepareSegments("target").get();
    ShardSegments segmentsStats = targetSegStats.getIndices().get("target").getShards().get(0).getShards()[0];
    assertTrue(segmentsStats.getNumberOfCommitted() > 0);
    assertNotEquals(segmentsStats.getSegments(), segmentsStats.getNumberOfCommitted());
    Iterable<IndicesService> dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class);
    for (IndicesService service : dataNodeInstances) {
        if (service.hasIndex(target.getIndex())) {
            IndexService indexShards = service.indexService(target.getIndex());
            IndexShard shard = indexShards.getShard(0);
            assertTrue(shard.isActive());
            shard.flushOnIdle(0);
            assertFalse(shard.isActive());
        }
    }
    assertBusy(() -> {
        IndicesSegmentResponse targetStats = client().admin().indices().prepareSegments("target").get();
        ShardSegments targetShardSegments = targetStats.getIndices().get("target").getShards().get(0).getShards()[0];
        Map<Integer, IndexShardSegments> source = sourceStats.getIndices().get("source").getShards();
        int numSourceSegments = 0;
        for (IndexShardSegments s : source.values()) {
            numSourceSegments += s.getAt(0).getNumberOfCommitted();
        }
        assertTrue(targetShardSegments.getSegments().size() < numSourceSegments);
        assertEquals(targetShardSegments.getNumberOfCommitted(), targetShardSegments.getNumberOfSearch());
        assertEquals(targetShardSegments.getNumberOfCommitted(), targetShardSegments.getSegments().size());
        assertEquals(1, targetShardSegments.getSegments().size());
    });
    // clean up
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)).get();
}

Example 12 with IndexService

use of org.opensearch.index.IndexService in project OpenSearch by opensearch-project.

the class IndexShardIT method testMaybeFlush.

public void testMaybeFlush() throws Exception {
    createIndex("test", Settings.builder().put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), Translog.Durability.REQUEST).build());
    ensureGreen();
    IndicesService indicesService = getInstanceFromNode(IndicesService.class);
    IndexService test = indicesService.indexService(resolveIndex("test"));
    IndexShard shard = test.getShardOrNull(0);
    assertFalse(shard.shouldPeriodicallyFlush());
    client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(135, /* size of the operation + one generation header&footer*/
    ByteSizeUnit.BYTES)).build()).get();
    client().prepareIndex("test").setId("0").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
    assertFalse(shard.shouldPeriodicallyFlush());
    shard.applyIndexOperationOnPrimary(Versions.MATCH_ANY, VersionType.INTERNAL, new SourceToParse("test", "_doc", "1", new BytesArray("{}"), XContentType.JSON), SequenceNumbers.UNASSIGNED_SEQ_NO, 0, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, false);
    assertTrue(shard.shouldPeriodicallyFlush());
    final Translog translog = getTranslog(shard);
    assertEquals(2, translog.stats().getUncommittedOperations());
    assertThat(shard.flushStats().getTotal(), equalTo(0L));
    client().prepareIndex("test").setId("2").setSource("{}", XContentType.JSON).setRefreshPolicy(randomBoolean() ? IMMEDIATE : NONE).get();
    assertThat(shard.getLastKnownGlobalCheckpoint(), equalTo(2L));
    assertBusy(() -> {
        // this is async
        assertFalse(shard.shouldPeriodicallyFlush());
        assertThat(shard.flushStats().getPeriodic(), equalTo(1L));
        assertThat(shard.flushStats().getTotal(), equalTo(1L));
    });
    shard.sync();
    assertThat(shard.getLastSyncedGlobalCheckpoint(), equalTo(2L));
    assertThat("last commit [" + shard.commitStats().getUserData() + "]", translog.stats().getUncommittedOperations(), equalTo(0));
    long size = Math.max(translog.stats().getUncommittedSizeInBytes(), Translog.DEFAULT_HEADER_SIZE_IN_BYTES + 1);
    logger.info("--> current translog size: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration());
    client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), new ByteSizeValue(size, ByteSizeUnit.BYTES)).build()).get();
    client().prepareDelete("test", "2").get();
    logger.info("--> translog size after delete: [{}] num_ops [{}] generation [{}]", translog.stats().getUncommittedSizeInBytes(), translog.stats().getUncommittedOperations(), translog.getGeneration());
    assertBusy(() -> {
        // this is async
        final TranslogStats translogStats = translog.stats();
        final CommitStats commitStats = shard.commitStats();
        final FlushStats flushStats = shard.flushStats();
        logger.info("--> translog stats [{}] gen [{}] commit_stats [{}] flush_stats [{}/{}]", Strings.toString(translogStats), translog.getGeneration().translogFileGeneration, commitStats.getUserData(), flushStats.getPeriodic(), flushStats.getTotal());
        assertFalse(shard.shouldPeriodicallyFlush());
    });
    shard.sync();
    assertEquals(0, translog.stats().getUncommittedOperations());
}

Example 13 with IndexService

use of org.opensearch.index.IndexService in project OpenSearch by opensearch-project.

the class IndexShardIT method testUpdatePriority.

public void testUpdatePriority() {
    assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put(IndexMetadata.SETTING_PRIORITY, 200)));
    IndexService indexService = getInstanceFromNode(IndicesService.class).indexService(resolveIndex("test"));
    assertEquals(200, indexService.getIndexSettings().getSettings().getAsInt(IndexMetadata.SETTING_PRIORITY, 0).intValue());
    client().admin().indices().prepareUpdateSettings("test").setSettings(Settings.builder().put(IndexMetadata.SETTING_PRIORITY, 400).build()).get();
    assertEquals(400, indexService.getIndexSettings().getSettings().getAsInt(IndexMetadata.SETTING_PRIORITY, 0).intValue());
}

Example 14 with IndexService

use of org.opensearch.index.IndexService in project OpenSearch by opensearch-project.

the class IndexShardIT method testShardChangesWithDefaultDocType.

public void testShardChangesWithDefaultDocType() throws Exception {
    Settings settings = Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).put("index.translog.flush_threshold_size", // do not flush
    "512mb").put("index.soft_deletes.enabled", true).build();
    IndexService indexService = createIndex("index", settings, "user_doc", "title", "type=keyword");
    int numOps = between(1, 10);
    for (int i = 0; i < numOps; i++) {
        if (randomBoolean()) {
            client().prepareIndex("index").setId(randomFrom("1", "2")).setSource("{}", XContentType.JSON).get();
        } else {
            client().prepareDelete("index", randomFrom("1", "2")).get();
        }
    }
    IndexShard shard = indexService.getShard(0);
    try (Translog.Snapshot luceneSnapshot = shard.newChangesSnapshot("test", 0, numOps - 1, true);
        Translog.Snapshot translogSnapshot = getTranslog(shard).newSnapshot()) {
        List<Translog.Operation> opsFromLucene = TestTranslog.drainSnapshot(luceneSnapshot, true);
        List<Translog.Operation> opsFromTranslog = TestTranslog.drainSnapshot(translogSnapshot, true);
        assertThat(opsFromLucene, equalTo(opsFromTranslog));
    }
}

Example 15 with IndexService

use of org.opensearch.index.IndexService in project OpenSearch by opensearch-project.

the class GlobalCheckpointSyncIT method runGlobalCheckpointSyncTest.

private void runGlobalCheckpointSyncTest(final TimeValue globalCheckpointSyncInterval, final Consumer<Client> beforeIndexing, final Consumer<Client> afterIndexing) throws Exception {
    final int numberOfReplicas = randomIntBetween(1, 4);
    internalCluster().ensureAtLeastNumDataNodes(1 + numberOfReplicas);
    prepareCreate("test", Settings.builder().put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), globalCheckpointSyncInterval).put("index.number_of_replicas", numberOfReplicas)).get();
    if (randomBoolean()) {
        ensureGreen();
    }
    beforeIndexing.accept(client());
    final int numberOfDocuments = randomIntBetween(0, 256);
    final int numberOfThreads = randomIntBetween(1, 4);
    final CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
    // start concurrent indexing threads
    final List<Thread> threads = new ArrayList<>(numberOfThreads);
    for (int i = 0; i < numberOfThreads; i++) {
        final int index = i;
        final Thread thread = new Thread(() -> {
            try {
                barrier.await();
            } catch (BrokenBarrierException | InterruptedException e) {
                throw new RuntimeException(e);
            }
            for (int j = 0; j < numberOfDocuments; j++) {
                final String id = Integer.toString(index * numberOfDocuments + j);
                client().prepareIndex("test").setId(id).setSource("{\"foo\": " + id + "}", XContentType.JSON).get();
            }
            try {
                barrier.await();
            } catch (BrokenBarrierException | InterruptedException e) {
                throw new RuntimeException(e);
            }
        });
        threads.add(thread);
        thread.start();
    }
    // synchronize the start of the threads
    barrier.await();
    // wait for the threads to finish
    barrier.await();
    afterIndexing.accept(client());
    assertBusy(() -> {
        for (IndicesService indicesService : internalCluster().getDataNodeInstances(IndicesService.class)) {
            for (IndexService indexService : indicesService) {
                for (IndexShard shard : indexService) {
                    if (shard.routingEntry().primary()) {
                        final SeqNoStats seqNoStats = shard.seqNoStats();
                        assertThat("shard " + shard.routingEntry() + " seq_no [" + seqNoStats + "]", seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
                    }
                }
            }
        }
    }, 60, TimeUnit.SECONDS);
    ensureGreen("test");
    for (final Thread thread : threads) {
        thread.join();
    }
}