Examples with IndicesService org.opensearch.indices.IndicesService used on opensource projects

Example 11 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class IndexShard method startRecovery.

public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService, RecoveryListener recoveryListener, RepositoriesService repositoriesService, Consumer<MappingMetadata> mappingUpdateConsumer, IndicesService indicesService) {
    // }
    assert recoveryState.getRecoverySource().equals(shardRouting.recoverySource());
    switch(recoveryState.getRecoverySource().getType()) {
        case EMPTY_STORE:
        case EXISTING_STORE:
            executeRecovery("from store", recoveryState, recoveryListener, this::recoverFromStore);
            break;
        case PEER:
            try {
                markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState);
                recoveryTargetService.startRecovery(this, recoveryState.getSourceNode(), recoveryListener);
            } catch (Exception e) {
                failShard("corrupted preexisting index", e);
                recoveryListener.onFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
            }
            break;
        case SNAPSHOT:
            final String repo = ((SnapshotRecoverySource) recoveryState.getRecoverySource()).snapshot().getRepository();
            executeRecovery("from snapshot", recoveryState, recoveryListener, l -> restoreFromRepository(repositoriesService.repository(repo), l));
            break;
        case LOCAL_SHARDS:
            final IndexMetadata indexMetadata = indexSettings().getIndexMetadata();
            final Index resizeSourceIndex = indexMetadata.getResizeSourceIndex();
            final List<IndexShard> startedShards = new ArrayList<>();
            final IndexService sourceIndexService = indicesService.indexService(resizeSourceIndex);
            final Set<ShardId> requiredShards;
            final int numShards;
            if (sourceIndexService != null) {
                requiredShards = IndexMetadata.selectRecoverFromShards(shardId().id(), sourceIndexService.getMetadata(), indexMetadata.getNumberOfShards());
                for (IndexShard shard : sourceIndexService) {
                    if (shard.state() == IndexShardState.STARTED && requiredShards.contains(shard.shardId())) {
                        startedShards.add(shard);
                    }
                }
                numShards = requiredShards.size();
            } else {
                numShards = -1;
                requiredShards = Collections.emptySet();
            }
            if (numShards == startedShards.size()) {
                assert requiredShards.isEmpty() == false;
                executeRecovery("from local shards", recoveryState, recoveryListener, l -> recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream().filter((s) -> requiredShards.contains(s.shardId())).collect(Collectors.toList()), l));
            } else {
                final RuntimeException e;
                if (numShards == -1) {
                    e = new IndexNotFoundException(resizeSourceIndex);
                } else {
                    e = new IllegalStateException("not all required shards of index " + resizeSourceIndex + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId());
                }
                throw e;
            }
            break;
        default:
            throw new IllegalArgumentException("Unknown recovery source " + recoveryState.getRecoverySource());
    }
}

Example 12 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class GetIndexActionTests method setUp.

@Before
public void setUp() throws Exception {
    super.setUp();
    settingsFilter = new SettingsModule(Settings.EMPTY, emptyList(), emptyList(), emptySet()).getSettingsFilter();
    threadPool = new TestThreadPool("GetIndexActionTests");
    clusterService = getInstanceFromNode(ClusterService.class);
    indicesService = getInstanceFromNode(IndicesService.class);
    CapturingTransport capturingTransport = new CapturingTransport();
    transportService = capturingTransport.createTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> clusterService.localNode(), null, emptySet());
    transportService.start();
    transportService.acceptIncomingRequests();
    getIndexAction = new GetIndexActionTests.TestTransportGetIndexAction();
}

Example 13 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class ShrinkIndexIT method testShrinkCommitsMergeOnIdle.

public void testShrinkCommitsMergeOnIdle() throws Exception {
    prepareCreate("source").setSettings(Settings.builder().put(indexSettings()).put("index.number_of_replicas", 0).put("number_of_shards", 5)).get();
    for (int i = 0; i < 30; i++) {
        client().prepareIndex("source").setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get();
    }
    client().admin().indices().prepareFlush("source").get();
    ImmutableOpenMap<String, DiscoveryNode> dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes();
    DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class);
    // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node
    // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due
    // to the require._name below.
    ensureGreen();
    // relocate all shards to one node such that we can merge it.
    client().admin().indices().prepareUpdateSettings("source").setSettings(Settings.builder().put("index.routing.allocation.require._name", discoveryNodes[0].getName()).put("index.blocks.write", true)).get();
    ensureGreen();
    IndicesSegmentResponse sourceStats = client().admin().indices().prepareSegments("source").get();
    // disable rebalancing to be able to capture the right stats. balancing can move the target primary
    // making it hard to pin point the source shards.
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")).get();
    // now merge source into a single shard index
    assertAcked(client().admin().indices().prepareResizeIndex("source", "target").setSettings(Settings.builder().put("index.number_of_replicas", 0).build()).get());
    ensureGreen();
    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
    IndexMetadata target = clusterStateResponse.getState().getMetadata().index("target");
    client().admin().indices().prepareForceMerge("target").setMaxNumSegments(1).setFlush(false).get();
    IndicesSegmentResponse targetSegStats = client().admin().indices().prepareSegments("target").get();
    ShardSegments segmentsStats = targetSegStats.getIndices().get("target").getShards().get(0).getShards()[0];
    assertTrue(segmentsStats.getNumberOfCommitted() > 0);
    assertNotEquals(segmentsStats.getSegments(), segmentsStats.getNumberOfCommitted());
    Iterable<IndicesService> dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class);
    for (IndicesService service : dataNodeInstances) {
        if (service.hasIndex(target.getIndex())) {
            IndexService indexShards = service.indexService(target.getIndex());
            IndexShard shard = indexShards.getShard(0);
            assertTrue(shard.isActive());
            shard.flushOnIdle(0);
            assertFalse(shard.isActive());
        }
    }
    assertBusy(() -> {
        IndicesSegmentResponse targetStats = client().admin().indices().prepareSegments("target").get();
        ShardSegments targetShardSegments = targetStats.getIndices().get("target").getShards().get(0).getShards()[0];
        Map<Integer, IndexShardSegments> source = sourceStats.getIndices().get("source").getShards();
        int numSourceSegments = 0;
        for (IndexShardSegments s : source.values()) {
            numSourceSegments += s.getAt(0).getNumberOfCommitted();
        }
        assertTrue(targetShardSegments.getSegments().size() < numSourceSegments);
        assertEquals(targetShardSegments.getNumberOfCommitted(), targetShardSegments.getNumberOfSearch());
        assertEquals(targetShardSegments.getNumberOfCommitted(), targetShardSegments.getSegments().size());
        assertEquals(1, targetShardSegments.getSegments().size());
    });
    // clean up
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)).get();
}

Example 14 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class TransportSearchIT method testSearchIdle.

public void testSearchIdle() throws Exception {
    int numOfReplicas = randomIntBetween(0, 1);
    internalCluster().ensureAtLeastNumDataNodes(numOfReplicas + 1);
    final Settings.Builder settings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 5)).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas).put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.timeValueMillis(randomIntBetween(50, 500)));
    assertAcked(prepareCreate("test").setSettings(settings).setMapping("created_date", "type=date,format=yyyy-MM-dd"));
    ensureGreen("test");
    assertBusy(() -> {
        for (String node : internalCluster().nodesInclude("test")) {
            final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
            for (IndexShard indexShard : indicesService.indexServiceSafe(resolveIndex("test"))) {
                assertTrue(indexShard.isSearchIdle());
            }
        }
    });
    client().prepareIndex("test").setId("1").setSource("created_date", "2020-01-01").get();
    client().prepareIndex("test").setId("2").setSource("created_date", "2020-01-02").get();
    client().prepareIndex("test").setId("3").setSource("created_date", "2020-01-03").get();
    assertBusy(() -> {
        SearchResponse resp = client().prepareSearch("test").setQuery(new RangeQueryBuilder("created_date").gte("2020-01-02").lte("2020-01-03")).setPreFilterShardSize(randomIntBetween(1, 3)).get();
        assertThat(resp.getHits().getTotalHits().value, equalTo(2L));
    });
}

Example 15 with IndicesService

use of org.opensearch.indices.IndicesService in project OpenSearch by opensearch-project.

the class MetadataIndexTemplateService method addComponentTemplate.

// Package visible for testing
ClusterState addComponentTemplate(final ClusterState currentState, final boolean create, final String name, final ComponentTemplate template) throws Exception {
    final ComponentTemplate existing = currentState.metadata().componentTemplates().get(name);
    if (create && existing != null) {
        throw new IllegalArgumentException("component template [" + name + "] already exists");
    }
    CompressedXContent mappings = template.template().mappings();
    String stringMappings = mappings == null ? null : mappings.string();
    // We may need to normalize index settings, so do that also
    Settings finalSettings = template.template().settings();
    if (finalSettings != null) {
        finalSettings = Settings.builder().put(finalSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build();
    }
    // Collect all the composable (index) templates that use this component template, we'll use
    // this for validating that they're still going to be valid after this component template
    // has been updated
    final Map<String, ComposableIndexTemplate> templatesUsingComponent = currentState.metadata().templatesV2().entrySet().stream().filter(e -> e.getValue().composedOf().contains(name)).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    // if we're updating a component template, let's check if it's part of any V2 template that will yield the CT update invalid
    if (create == false && finalSettings != null) {
        // if the CT is specifying the `index.hidden` setting it cannot be part of any global template
        if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(finalSettings)) {
            List<String> globalTemplatesThatUseThisComponent = new ArrayList<>();
            for (Map.Entry<String, ComposableIndexTemplate> entry : templatesUsingComponent.entrySet()) {
                ComposableIndexTemplate templateV2 = entry.getValue();
                if (templateV2.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) {
                    // global templates don't support configuring the `index.hidden` setting so we don't need to resolve the settings as
                    // no other component template can remove this setting from the resolved settings, so just invalidate this update
                    globalTemplatesThatUseThisComponent.add(entry.getKey());
                }
            }
            if (globalTemplatesThatUseThisComponent.isEmpty() == false) {
                throw new IllegalArgumentException("cannot update component template [" + name + "] because the following global templates would resolve to specifying the [" + IndexMetadata.SETTING_INDEX_HIDDEN + "] setting: [" + String.join(",", globalTemplatesThatUseThisComponent) + "]");
            }
        }
    }
    // Mappings in component templates don't include _doc, so update the mappings to include this single type
    if (stringMappings != null) {
        Map<String, Object> parsedMappings = MapperService.parseMapping(xContentRegistry, stringMappings);
        if (parsedMappings.size() > 0) {
            stringMappings = Strings.toString(XContentFactory.jsonBuilder().startObject().field(MapperService.SINGLE_MAPPING_NAME, parsedMappings).endObject());
        }
    }
    final Template finalTemplate = new Template(finalSettings, stringMappings == null ? null : new CompressedXContent(stringMappings), template.template().aliases());
    final ComponentTemplate finalComponentTemplate = new ComponentTemplate(finalTemplate, template.version(), template.metadata());
    if (finalComponentTemplate.equals(existing)) {
        return currentState;
    }
    validateTemplate(finalSettings, stringMappings, indicesService, xContentRegistry);
    validate(name, finalComponentTemplate);
    // Validate all composable index templates that use this component template
    if (templatesUsingComponent.size() > 0) {
        ClusterState tempStateWithComponentTemplateAdded = ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate)).build();
        Exception validationFailure = null;
        for (Map.Entry<String, ComposableIndexTemplate> entry : templatesUsingComponent.entrySet()) {
            final String composableTemplateName = entry.getKey();
            final ComposableIndexTemplate composableTemplate = entry.getValue();
            try {
                validateCompositeTemplate(tempStateWithComponentTemplateAdded, composableTemplateName, composableTemplate, indicesService, xContentRegistry);
            } catch (Exception e) {
                if (validationFailure == null) {
                    validationFailure = new IllegalArgumentException("updating component template [" + name + "] results in invalid composable template [" + composableTemplateName + "] after templates are merged", e);
                } else {
                    validationFailure.addSuppressed(e);
                }
            }
        }
        if (validationFailure != null) {
            throw validationFailure;
        }
    }
    logger.info("{} component template [{}]", existing == null ? "adding" : "updating", name);
    return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.metadata()).put(name, finalComponentTemplate)).build();
}