Examples with IndexSettings org.opensearch.index.IndexSettings used on opensource projects

Example 71 with IndexSettings

use of org.opensearch.index.IndexSettings in project OpenSearch by opensearch-project.

the class IndexShard method maybeSyncGlobalCheckpoint.

/**
 * Syncs the global checkpoint to the replicas if the global checkpoint on at least one replica is behind the global checkpoint on the
 * primary.
 */
public void maybeSyncGlobalCheckpoint(final String reason) {
    verifyNotClosed();
    assert shardRouting.primary() : "only call maybeSyncGlobalCheckpoint on primary shard";
    if (replicationTracker.isPrimaryMode() == false) {
        return;
    }
    assert assertPrimaryMode();
    // only sync if there are no operations in flight, or when using async durability
    final SeqNoStats stats = getEngine().getSeqNoStats(replicationTracker.getGlobalCheckpoint());
    final boolean asyncDurability = indexSettings().getTranslogDurability() == Translog.Durability.ASYNC;
    if (stats.getMaxSeqNo() == stats.getGlobalCheckpoint() || asyncDurability) {
        final ObjectLongMap<String> globalCheckpoints = getInSyncGlobalCheckpoints();
        final long globalCheckpoint = replicationTracker.getGlobalCheckpoint();
        // async durability means that the local checkpoint might lag (as it is only advanced on fsync)
        // periodically ask for the newest local checkpoint by syncing the global checkpoint, so that ultimately the global
        // checkpoint can be synced. Also take into account that a shard might be pending sync, which means that it isn't
        // in the in-sync set just yet but might be blocked on waiting for its persisted local checkpoint to catch up to
        // the global checkpoint.
        final boolean syncNeeded = (asyncDurability && (stats.getGlobalCheckpoint() < stats.getMaxSeqNo() || replicationTracker.pendingInSync())) || // check if the persisted global checkpoint
        StreamSupport.stream(globalCheckpoints.values().spliterator(), false).anyMatch(v -> v.value < globalCheckpoint);
        // only sync if index is not closed and there is a shard lagging the primary
        if (syncNeeded && indexSettings.getIndexMetadata().getState() == IndexMetadata.State.OPEN) {
            logger.trace("syncing global checkpoint for [{}]", reason);
            globalCheckpointSyncer.run();
        }
    }
}

Example 72 with IndexSettings

use of org.opensearch.index.IndexSettings in project OpenSearch by opensearch-project.

the class RemoveCorruptedShardDataCommand method findAndProcessShardPath.

protected void findAndProcessShardPath(OptionSet options, Environment environment, Path[] dataPaths, int nodeLockId, ClusterState clusterState, CheckedConsumer<ShardPath, IOException> consumer) throws IOException {
    final Settings settings = environment.settings();
    final IndexMetadata indexMetadata;
    final int shardId;
    final int fromNodeId;
    final int toNodeId;
    if (options.has(folderOption)) {
        final Path path = getPath(folderOption.value(options)).getParent();
        final Path shardParent = path.getParent();
        final Path shardParentParent = shardParent.getParent();
        final Path indexPath = path.resolve(ShardPath.INDEX_FOLDER_NAME);
        if (Files.exists(indexPath) == false || Files.isDirectory(indexPath) == false) {
            throw new OpenSearchException("index directory [" + indexPath + "], must exist and be a directory");
        }
        final String shardIdFileName = path.getFileName().toString();
        final String nodeIdFileName = shardParentParent.getParent().getFileName().toString();
        final String indexUUIDFolderName = shardParent.getFileName().toString();
        if (Files.isDirectory(path) && // SHARD-ID path element check
        shardIdFileName.chars().allMatch(Character::isDigit) && // `indices` check
        NodeEnvironment.INDICES_FOLDER.equals(shardParentParent.getFileName().toString()) && // NODE-ID check
        nodeIdFileName.chars().allMatch(Character::isDigit) && // `nodes` check
        NodeEnvironment.NODES_FOLDER.equals(shardParentParent.getParent().getParent().getFileName().toString())) {
            shardId = Integer.parseInt(shardIdFileName);
            fromNodeId = Integer.parseInt(nodeIdFileName);
            toNodeId = fromNodeId + 1;
            indexMetadata = StreamSupport.stream(clusterState.metadata().indices().values().spliterator(), false).map(imd -> imd.value).filter(imd -> imd.getIndexUUID().equals(indexUUIDFolderName)).findFirst().orElse(null);
        } else {
            throw new OpenSearchException("Unable to resolve shard id. Wrong folder structure at [ " + path.toString() + " ], expected .../nodes/[NODE-ID]/indices/[INDEX-UUID]/[SHARD-ID]");
        }
    } else {
        // otherwise resolve shardPath based on the index name and shard id
        String indexName = Objects.requireNonNull(indexNameOption.value(options), "Index name is required");
        shardId = Objects.requireNonNull(shardIdOption.value(options), "Shard ID is required");
        indexMetadata = clusterState.metadata().index(indexName);
    }
    if (indexMetadata == null) {
        throw new OpenSearchException("Unable to find index in cluster state");
    }
    final IndexSettings indexSettings = new IndexSettings(indexMetadata, settings);
    final Index index = indexMetadata.getIndex();
    final ShardId shId = new ShardId(index, shardId);
    for (Path dataPath : dataPaths) {
        final Path shardPathLocation = dataPath.resolve(NodeEnvironment.INDICES_FOLDER).resolve(index.getUUID()).resolve(Integer.toString(shId.id()));
        if (Files.exists(shardPathLocation)) {
            final ShardPath shardPath = ShardPath.loadShardPath(logger, shId, indexSettings.customDataPath(), new Path[] { shardPathLocation }, nodeLockId, dataPath);
            if (shardPath != null) {
                consumer.accept(shardPath);
                return;
            }
        }
    }
    throw new OpenSearchException("Unable to resolve shard path for index [" + indexMetadata.getIndex().getName() + "] and shard id [" + shardId + "]");
}

Example 73 with IndexSettings

use of org.opensearch.index.IndexSettings in project OpenSearch by opensearch-project.

the class SettingTests method testLogSettingUpdate.

@TestLogging(value = "org.opensearch.common.settings.IndexScopedSettings:INFO", reason = "to ensure we log INFO-level messages from IndexScopedSettings")
public void testLogSettingUpdate() throws Exception {
    final IndexMetadata metadata = newIndexMeta("index1", Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "20s").build());
    final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
    final Logger logger = LogManager.getLogger(IndexScopedSettings.class);
    try (MockLogAppender mockLogAppender = MockLogAppender.createForLoggers(logger)) {
        mockLogAppender.addExpectation(new MockLogAppender.SeenEventExpectation("message", "org.opensearch.common.settings.IndexScopedSettings", Level.INFO, "updating [index.refresh_interval] from [20s] to [10s]") {

            @Override
            public boolean innerMatch(LogEvent event) {
                return event.getMarker().getName().equals(" [index1]");
            }
        });
        settings.updateIndexMetadata(newIndexMeta("index1", Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s").build()));
        mockLogAppender.assertAllExpectationsMatched();
    }
}

Example 74 with IndexSettings

use of org.opensearch.index.IndexSettings in project OpenSearch by opensearch-project.

the class IndicesClientIT method testSplit.

@SuppressWarnings("unchecked")
public void testSplit() throws IOException {
    createIndex("source", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).put("index.number_of_routing_shards", 4).build());
    updateIndexSettings("source", Settings.builder().put("index.blocks.write", true));
    ResizeRequest resizeRequest = new ResizeRequest("target", "source");
    resizeRequest.setResizeType(ResizeType.SPLIT);
    Settings targetSettings = Settings.builder().put("index.number_of_shards", 4).put("index.number_of_replicas", 0).build();
    resizeRequest.setTargetIndex(new org.opensearch.action.admin.indices.create.CreateIndexRequest("target").settings(targetSettings).alias(new Alias("alias")));
    ResizeResponse resizeResponse = execute(resizeRequest, highLevelClient().indices()::split, highLevelClient().indices()::splitAsync);
    assertTrue(resizeResponse.isAcknowledged());
    assertTrue(resizeResponse.isShardsAcknowledged());
    Map<String, Object> getIndexResponse = getAsMap("target");
    Map<String, Object> indexSettings = (Map<String, Object>) XContentMapValues.extractValue("target.settings.index", getIndexResponse);
    assertNotNull(indexSettings);
    assertEquals("4", indexSettings.get("number_of_shards"));
    assertEquals("0", indexSettings.get("number_of_replicas"));
    Map<String, Object> aliasData = (Map<String, Object>) XContentMapValues.extractValue("target.aliases.alias", getIndexResponse);
    assertNotNull(aliasData);
}

Example 75 with IndexSettings

use of org.opensearch.index.IndexSettings in project OpenSearch by opensearch-project.

the class IndicesClientIT method testClone.

@SuppressWarnings("unchecked")
public void testClone() throws IOException {
    createIndex("source", Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).put("index.number_of_routing_shards", 4).build());
    updateIndexSettings("source", Settings.builder().put("index.blocks.write", true));
    ResizeRequest resizeRequest = new ResizeRequest("target", "source");
    resizeRequest.setResizeType(ResizeType.CLONE);
    Settings targetSettings = Settings.builder().put("index.number_of_shards", 2).put("index.number_of_replicas", 0).build();
    resizeRequest.setTargetIndex(new org.opensearch.action.admin.indices.create.CreateIndexRequest("target").settings(targetSettings).alias(new Alias("alias")));
    ResizeResponse resizeResponse = execute(resizeRequest, highLevelClient().indices()::clone, highLevelClient().indices()::cloneAsync);
    assertTrue(resizeResponse.isAcknowledged());
    assertTrue(resizeResponse.isShardsAcknowledged());
    Map<String, Object> getIndexResponse = getAsMap("target");
    Map<String, Object> indexSettings = (Map<String, Object>) XContentMapValues.extractValue("target.settings.index", getIndexResponse);
    assertNotNull(indexSettings);
    assertEquals("2", indexSettings.get("number_of_shards"));
    assertEquals("0", indexSettings.get("number_of_replicas"));
    Map<String, Object> aliasData = (Map<String, Object>) XContentMapValues.extractValue("target.aliases.alias", getIndexResponse);
    assertNotNull(aliasData);
}