use of org.opensearch.Version in project OpenSearch by opensearch-project.
the class DiscoveryModuleTests method testLegacyHostsProvider.
public void testLegacyHostsProvider() {
Settings settings = Settings.builder().put(DiscoveryModule.LEGACY_DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "custom").build();
AtomicBoolean created = new AtomicBoolean(false);
DummyHostsProviderPlugin plugin = () -> Collections.singletonMap("custom", () -> {
created.set(true);
return hostsResolver -> Collections.emptyList();
});
newModule(settings, Collections.singletonList(plugin));
assertTrue(created.get());
assertWarnings("[discovery.zen.hosts_provider] setting was deprecated in OpenSearch and will be removed in a future release! " + "See the breaking changes documentation for the next major version.");
}
use of org.opensearch.Version in project OpenSearch by opensearch-project.
the class BlobStoreRepository method doGetRepositoryData.
private void doGetRepositoryData(ActionListener<RepositoryData> listener) {
// Retry loading RepositoryData in a loop in case we run into concurrent modifications of the repository.
// Keep track of the most recent generation we failed to load so we can break out of the loop if we fail to load the same
// generation repeatedly.
long lastFailedGeneration = RepositoryData.UNKNOWN_REPO_GEN;
while (true) {
final long genToLoad;
if (bestEffortConsistency) {
// We're only using #latestKnownRepoGen as a hint in this mode and listing repo contents as a secondary way of trying
// to find a higher generation
final long generation;
try {
generation = latestIndexBlobId();
} catch (IOException ioe) {
listener.onFailure(new RepositoryException(metadata.name(), "Could not determine repository generation from root blobs", ioe));
return;
}
genToLoad = latestKnownRepoGen.updateAndGet(known -> Math.max(known, generation));
if (genToLoad > generation) {
logger.info("Determined repository generation [" + generation + "] from repository contents but correct generation must be at least [" + genToLoad + "]");
}
} else {
// We only rely on the generation tracked in #latestKnownRepoGen which is exclusively updated from the cluster state
genToLoad = latestKnownRepoGen.get();
}
try {
final Tuple<Long, BytesReference> cached = latestKnownRepositoryData.get();
final RepositoryData loaded;
// Caching is not used with #bestEffortConsistency see docs on #cacheRepositoryData for details
if (bestEffortConsistency == false && cached != null && cached.v1() == genToLoad) {
loaded = repositoryDataFromCachedEntry(cached);
} else {
loaded = getRepositoryData(genToLoad);
// We can cache serialized in the most recent version here without regard to the actual repository metadata version
// since we're only caching the information that we just wrote and thus won't accidentally cache any information that
// isn't safe
cacheRepositoryData(BytesReference.bytes(loaded.snapshotsToXContent(XContentFactory.jsonBuilder(), Version.CURRENT)), genToLoad);
}
listener.onResponse(loaded);
return;
} catch (RepositoryException e) {
// If the generation to load changed concurrently and we didn't just try loading the same generation before we retry
if (genToLoad != latestKnownRepoGen.get() && genToLoad != lastFailedGeneration) {
lastFailedGeneration = genToLoad;
logger.warn("Failed to load repository data generation [" + genToLoad + "] because a concurrent operation moved the current generation to [" + latestKnownRepoGen.get() + "]", e);
continue;
}
if (bestEffortConsistency == false && ExceptionsHelper.unwrap(e, NoSuchFileException.class) != null) {
// We did not find the expected index-N even though the cluster state continues to point at the missing value
// of N so we mark this repository as corrupted.
markRepoCorrupted(genToLoad, e, ActionListener.wrap(v -> listener.onFailure(corruptedStateException(e)), listener::onFailure));
} else {
listener.onFailure(e);
}
return;
} catch (Exception e) {
listener.onFailure(new RepositoryException(metadata.name(), "Unexpected exception when loading repository data", e));
return;
}
}
}
use of org.opensearch.Version in project OpenSearch by opensearch-project.
the class RepositoryData method parseSnapshots.
/**
* Parses the "snapshots" field and fills maps for the various per snapshot properties. This method must run before
* {@link #parseIndices} which will rely on the maps of snapshot properties to have been populated already.
*
* @param parser x-content parse
* @param snapshots map of snapshot uuid to {@link SnapshotId}
* @param snapshotStates map of snapshot uuid to {@link SnapshotState}
* @param snapshotVersions map of snapshot uuid to {@link Version} that the snapshot was taken in
* @param indexMetaLookup map of {@link SnapshotId} to map of index id (as returned by {@link IndexId#getId}) that defines the index
* metadata generations for the snapshot
*/
private static void parseSnapshots(XContentParser parser, Map<String, SnapshotId> snapshots, Map<String, SnapshotState> snapshotStates, Map<String, Version> snapshotVersions, Map<SnapshotId, Map<String, String>> indexMetaLookup) throws IOException {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser);
final Map<String, String> stringDeduplicator = new HashMap<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
String name = null;
String uuid = null;
SnapshotState state = null;
Map<String, String> metaGenerations = null;
Version version = null;
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
String currentFieldName = parser.currentName();
parser.nextToken();
switch(currentFieldName) {
case NAME:
name = parser.text();
break;
case UUID:
uuid = parser.text();
break;
case STATE:
state = SnapshotState.fromValue((byte) parser.intValue());
break;
case INDEX_METADATA_LOOKUP:
metaGenerations = parser.map(HashMap::new, p -> stringDeduplicator.computeIfAbsent(p.text(), Function.identity()));
break;
case VERSION:
version = Version.fromString(parser.text());
break;
}
}
final SnapshotId snapshotId = new SnapshotId(name, uuid);
if (state != null) {
snapshotStates.put(uuid, state);
}
if (version != null) {
snapshotVersions.put(uuid, version);
}
snapshots.put(uuid, snapshotId);
if (metaGenerations != null && metaGenerations.isEmpty() == false) {
indexMetaLookup.put(snapshotId, metaGenerations);
}
}
}
use of org.opensearch.Version in project OpenSearch by opensearch-project.
the class RepositoryData method snapshotsFromXContent.
/**
* Reads an instance of {@link RepositoryData} from x-content, loading the snapshots and indices metadata.
*
* @param fixBrokenShardGens set to {@code true} to filter out broken shard generations read from the {@code parser} via
* {@link ShardGenerations#fixShardGeneration}. Used to disable fixing broken generations when reading
* from cached bytes that we trust to not contain broken generations.
*/
public static RepositoryData snapshotsFromXContent(XContentParser parser, long genId, boolean fixBrokenShardGens) throws IOException {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
final Map<String, SnapshotId> snapshots = new HashMap<>();
final Map<String, SnapshotState> snapshotStates = new HashMap<>();
final Map<String, Version> snapshotVersions = new HashMap<>();
final Map<IndexId, List<SnapshotId>> indexSnapshots = new HashMap<>();
final Map<String, IndexId> indexLookup = new HashMap<>();
final ShardGenerations.Builder shardGenerations = ShardGenerations.builder();
final Map<SnapshotId, Map<String, String>> indexMetaLookup = new HashMap<>();
Map<String, String> indexMetaIdentifiers = null;
while (parser.nextToken() == XContentParser.Token.FIELD_NAME) {
final String field = parser.currentName();
switch(field) {
case SNAPSHOTS:
parseSnapshots(parser, snapshots, snapshotStates, snapshotVersions, indexMetaLookup);
break;
case INDICES:
parseIndices(parser, fixBrokenShardGens, snapshots, indexSnapshots, indexLookup, shardGenerations);
break;
case INDEX_METADATA_IDENTIFIERS:
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
indexMetaIdentifiers = parser.mapStrings();
break;
case MIN_VERSION:
XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_STRING, parser.nextToken(), parser);
final Version version = Version.fromString(parser.text());
assert SnapshotsService.useShardGenerations(version);
break;
default:
XContentParserUtils.throwUnknownField(field, parser.getTokenLocation());
}
}
return new RepositoryData(genId, snapshots, snapshotStates, snapshotVersions, indexSnapshots, shardGenerations.build(), buildIndexMetaGenerations(indexMetaLookup, indexLookup, indexMetaIdentifiers));
}
use of org.opensearch.Version in project OpenSearch by opensearch-project.
the class RepositoryData method snapshotsToXContent.
/**
* Writes the snapshots metadata and the related indices metadata to x-content.
*/
public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final Version repoMetaVersion) throws IOException {
builder.startObject();
// write the snapshots list
builder.startArray(SNAPSHOTS);
final boolean shouldWriteIndexGens = SnapshotsService.useIndexGenerations(repoMetaVersion);
final boolean shouldWriteShardGens = SnapshotsService.useShardGenerations(repoMetaVersion);
for (final SnapshotId snapshot : getSnapshotIds()) {
builder.startObject();
builder.field(NAME, snapshot.getName());
final String snapshotUUID = snapshot.getUUID();
builder.field(UUID, snapshotUUID);
final SnapshotState state = snapshotStates.get(snapshotUUID);
if (state != null) {
builder.field(STATE, state.value());
}
if (shouldWriteIndexGens) {
builder.startObject(INDEX_METADATA_LOOKUP);
for (Map.Entry<IndexId, String> entry : indexMetaDataGenerations.lookup.getOrDefault(snapshot, Collections.emptyMap()).entrySet()) {
builder.field(entry.getKey().getId(), entry.getValue());
}
builder.endObject();
}
final Version version = snapshotVersions.get(snapshotUUID);
if (version != null) {
builder.field(VERSION, version.toString());
}
builder.endObject();
}
builder.endArray();
// write the indices map
builder.startObject(INDICES);
for (final IndexId indexId : getIndices().values()) {
builder.startObject(indexId.getName());
builder.field(INDEX_ID, indexId.getId());
builder.startArray(SNAPSHOTS);
List<SnapshotId> snapshotIds = indexSnapshots.get(indexId);
assert snapshotIds != null;
for (final SnapshotId snapshotId : snapshotIds) {
builder.value(snapshotId.getUUID());
}
builder.endArray();
if (shouldWriteShardGens) {
builder.startArray(SHARD_GENERATIONS);
for (String gen : shardGenerations.getGens(indexId)) {
builder.value(gen);
}
builder.endArray();
}
builder.endObject();
}
builder.endObject();
if (shouldWriteIndexGens) {
builder.field(MIN_VERSION, SnapshotsService.INDEX_GEN_IN_REPO_DATA_VERSION.toString());
builder.field(INDEX_METADATA_IDENTIFIERS, indexMetaDataGenerations.identifiers);
} else if (shouldWriteShardGens) {
// Add min version field to make it impossible for older OpenSearch versions to deserialize this object
builder.field(MIN_VERSION, SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.toString());
}
builder.endObject();
return builder;
}
Aggregations