use of org.elasticsearch.repositories.Repository in project crate by crate.
the class BlobStoreRepository method writeIndexGen.
/**
* Writing a new index generation is a three step process.
* First, the {@link RepositoryMetadata} entry for this repository is set into a pending state by incrementing its
* pending generation {@code P} while its safe generation {@code N} remains unchanged.
* Second, the updated {@link RepositoryData} is written to generation {@code P + 1}.
* Lastly, the {@link RepositoryMetadata} entry for this repository is updated to the new generation {@code P + 1} and thus
* pending and safe generation are set to the same value marking the end of the update of the repository data.
*
* @param repositoryData RepositoryData to write
* @param expectedGen expected repository generation at the start of the operation
* @param writeShardGens whether to write {@link ShardGenerations} to the new {@link RepositoryData} blob
* @param listener completion listener
*/
protected void writeIndexGen(RepositoryData repositoryData, long expectedGen, boolean writeShardGens, ActionListener<Void> listener) {
// can not write to a read only repository
assert isReadOnly() == false;
final long currentGen = repositoryData.getGenId();
if (currentGen != expectedGen) {
// the index file was updated by a concurrent operation, so we were operating on stale
// repository data
listener.onFailure(new RepositoryException(metadata.name(), "concurrent modification of the index-N file, expected current generation [" + expectedGen + "], actual current generation [" + currentGen + "]"));
return;
}
// Step 1: Set repository generation state to the next possible pending generation
final StepListener<Long> setPendingStep = new StepListener<>();
clusterService.submitStateUpdateTask("set pending repository generation [" + metadata.name() + "][" + expectedGen + "]", new ClusterStateUpdateTask() {
private long newGen;
@Override
public ClusterState execute(ClusterState currentState) {
final RepositoryMetadata meta = getRepoMetadata(currentState);
final String repoName = metadata.name();
final long genInState = meta.generation();
final boolean uninitializedMeta = meta.generation() == RepositoryData.UNKNOWN_REPO_GEN || bestEffortConsistency;
if (uninitializedMeta == false && meta.pendingGeneration() != genInState) {
LOGGER.info("Trying to write new repository data over unfinished write, repo [{}] is at " + "safe generation [{}] and pending generation [{}]", meta.name(), genInState, meta.pendingGeneration());
}
assert expectedGen == RepositoryData.EMPTY_REPO_GEN || uninitializedMeta || expectedGen == meta.generation() : "Expected non-empty generation [" + expectedGen + "] does not match generation tracked in [" + meta + "]";
// If we run into the empty repo generation for the expected gen, the repo is assumed to have been cleared of
// all contents by an external process so we reset the safe generation to the empty generation.
final long safeGeneration = expectedGen == RepositoryData.EMPTY_REPO_GEN ? RepositoryData.EMPTY_REPO_GEN : (uninitializedMeta ? expectedGen : genInState);
// Regardless of whether or not the safe generation has been reset, the pending generation always increments so that
// even if a repository has been manually cleared of all contents we will never reuse the same repository generation.
// This is motivated by the consistency behavior the S3 based blob repository implementation has to support which does
// not offer any consistency guarantees when it comes to overwriting the same blob name with different content.
final long nextPendingGen = metadata.pendingGeneration() + 1;
newGen = uninitializedMeta ? Math.max(expectedGen + 1, nextPendingGen) : nextPendingGen;
assert newGen > latestKnownRepoGen.get() : "Attempted new generation [" + newGen + "] must be larger than latest known generation [" + latestKnownRepoGen.get() + "]";
return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.getMetadata()).putCustom(RepositoriesMetadata.TYPE, currentState.metadata().<RepositoriesMetadata>custom(RepositoriesMetadata.TYPE).withUpdatedGeneration(repoName, safeGeneration, newGen)).build()).build();
}
@Override
public void onFailure(String source, Exception e) {
listener.onFailure(new RepositoryException(metadata.name(), "Failed to execute cluster state update [" + source + "]", e));
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
setPendingStep.onResponse(newGen);
}
});
final StepListener<RepositoryData> filterRepositoryDataStep = new StepListener<>();
// Step 2: Write new index-N blob to repository and update index.latest
setPendingStep.whenComplete(newGen -> threadPool().executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.wrap(listener, l -> {
// BwC logic: Load snapshot version information if any snapshot is missing a version in RepositoryData so that the new
// RepositoryData contains a version for every snapshot
final List<SnapshotId> snapshotIdsWithoutVersion = repositoryData.getSnapshotIds().stream().filter(snapshotId -> repositoryData.getVersion(snapshotId) == null).collect(Collectors.toList());
if (snapshotIdsWithoutVersion.isEmpty() == false) {
final Map<SnapshotId, Version> updatedVersionMap = new ConcurrentHashMap<>();
final GroupedActionListener<Void> loadAllVersionsListener = new GroupedActionListener<>(ActionListener.runAfter(new ActionListener<Collection<Void>>() {
@Override
public void onResponse(Collection<Void> voids) {
LOGGER.info("Successfully loaded all snapshot's version information for {} from snapshot metadata", AllocationService.firstListElementsToCommaDelimitedString(snapshotIdsWithoutVersion, SnapshotId::toString, LOGGER.isDebugEnabled()));
}
@Override
public void onFailure(Exception e) {
LOGGER.warn("Failure when trying to load missing version information from snapshot metadata", e);
}
}, () -> filterRepositoryDataStep.onResponse(repositoryData.withVersions(updatedVersionMap))), snapshotIdsWithoutVersion.size());
for (SnapshotId snapshotId : snapshotIdsWithoutVersion) {
threadPool().executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.run(loadAllVersionsListener, () -> {
ActionListener<SnapshotInfo> snapshotInfoListener = ActionListener.delegateFailure(loadAllVersionsListener, (delegate, snapshotInfo) -> {
updatedVersionMap.put(snapshotId, snapshotInfo.version());
delegate.onResponse(null);
});
getSnapshotInfo(snapshotId, snapshotInfoListener);
}));
}
} else {
filterRepositoryDataStep.onResponse(repositoryData);
}
})), listener::onFailure);
filterRepositoryDataStep.whenComplete(filteredRepositoryData -> {
final long newGen = setPendingStep.result();
if (latestKnownRepoGen.get() >= newGen) {
throw new IllegalArgumentException("Tried writing generation [" + newGen + "] but repository is at least at generation [" + latestKnownRepoGen.get() + "] already");
}
// write the index file
final String indexBlob = INDEX_FILE_PREFIX + Long.toString(newGen);
LOGGER.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob);
writeAtomic(indexBlob, BytesReference.bytes(filteredRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), writeShardGens)), true);
// write the current generation to the index-latest file
final BytesReference genBytes;
try (BytesStreamOutput bStream = new BytesStreamOutput()) {
bStream.writeLong(newGen);
genBytes = bStream.bytes();
}
LOGGER.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen);
writeAtomic(INDEX_LATEST_BLOB, genBytes, false);
// Step 3: Update CS to reflect new repository generation.
clusterService.submitStateUpdateTask("set safe repository generation [" + metadata.name() + "][" + newGen + "]", new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) {
final RepositoryMetadata meta = getRepoMetadata(currentState);
if (meta.generation() != expectedGen) {
throw new IllegalStateException("Tried to update repo generation to [" + newGen + "] but saw unexpected generation in state [" + meta + "]");
}
if (meta.pendingGeneration() != newGen) {
throw new IllegalStateException("Tried to update from unexpected pending repo generation [" + meta.pendingGeneration() + "] after write to generation [" + newGen + "]");
}
return ClusterState.builder(currentState).metadata(Metadata.builder(currentState.getMetadata()).putCustom(RepositoriesMetadata.TYPE, currentState.metadata().<RepositoriesMetadata>custom(RepositoriesMetadata.TYPE).withUpdatedGeneration(metadata.name(), newGen, newGen)).build()).build();
}
@Override
public void onFailure(String source, Exception e) {
listener.onFailure(new RepositoryException(metadata.name(), "Failed to execute cluster state update [" + source + "]", e));
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.run(listener, () -> {
// Delete all now outdated index files up to 1000 blobs back from the new generation.
// If there are more than 1000 dangling index-N cleanup functionality on repo delete will take care of them.
// Deleting one older than the current expectedGen is done for BwC reasons as older versions used to keep
// two index-N blobs around.
final List<String> oldIndexN = LongStream.range(Math.max(Math.max(expectedGen - 1, 0), newGen - 1000), newGen).mapToObj(gen -> INDEX_FILE_PREFIX + gen).collect(Collectors.toList());
try {
blobContainer().deleteBlobsIgnoringIfNotExists(oldIndexN);
} catch (IOException e) {
LOGGER.warn("Failed to clean up old index blobs {}", oldIndexN);
}
}));
}
});
}, listener::onFailure);
}
use of org.elasticsearch.repositories.Repository in project crate by crate.
the class RestoreService method restoreSnapshot.
/**
* Restores snapshot specified in the restore request.
*
* @param request restore request
* @param listener restore listener
*/
public void restoreSnapshot(final RestoreRequest request, final ActionListener<RestoreCompletionResponse> listener) {
try {
// Read snapshot info and metadata from the repository
final String repositoryName = request.repositoryName;
Repository repository = repositoriesService.repository(repositoryName);
final StepListener<RepositoryData> repositoryDataListener = new StepListener<>();
repository.getRepositoryData(repositoryDataListener);
repositoryDataListener.whenComplete(repositoryData -> {
final String snapshotName = request.snapshotName;
final Optional<SnapshotId> matchingSnapshotId = repositoryData.getSnapshotIds().stream().filter(s -> snapshotName.equals(s.getName())).findFirst();
if (matchingSnapshotId.isPresent() == false) {
throw new SnapshotRestoreException(repositoryName, snapshotName, "snapshot does not exist");
}
final SnapshotId snapshotId = matchingSnapshotId.get();
repository.getSnapshotInfo(snapshotId, ActionListener.delegateFailure(listener, (delegate, snapshotInfo) -> {
final Snapshot snapshot = new Snapshot(repositoryName, snapshotId);
// Make sure that we can restore from this snapshot
validateSnapshotRestorable(repositoryName, snapshotInfo);
// Resolve the indices from the snapshot that need to be restored
final List<String> indicesInSnapshot = request.includeIndices() ? filterIndices(snapshotInfo.indices(), request.indices(), request.indicesOptions()) : List.of();
final StepListener<Metadata> globalMetadataListener = new StepListener<>();
if (request.includeCustomMetadata() || request.includeGlobalSettings() || request.allTemplates() || (request.templates() != null && request.templates().length > 0)) {
repository.getSnapshotGlobalMetadata(snapshotId, globalMetadataListener);
} else {
globalMetadataListener.onResponse(Metadata.EMPTY_METADATA);
}
globalMetadataListener.whenComplete(globalMetadata -> {
var metadataBuilder = Metadata.builder(globalMetadata);
var indexIdsInSnapshot = repositoryData.resolveIndices(indicesInSnapshot);
var snapshotIndexMetadataListener = new StepListener<Collection<IndexMetadata>>();
repository.getSnapshotIndexMetadata(snapshotId, indexIdsInSnapshot, snapshotIndexMetadataListener);
snapshotIndexMetadataListener.whenComplete(snapshotIndexMetadata -> {
for (IndexMetadata indexMetadata : snapshotIndexMetadata) {
metadataBuilder.put(indexMetadata, false);
}
final Metadata metadata = metadataBuilder.build();
// Apply renaming on index names, returning a map of names where
// the key is the renamed index and the value is the original name
final Map<String, String> indices = renamedIndices(request, indicesInSnapshot);
// Now we can start the actual restore process by adding shards to be recovered in the cluster state
// and updating cluster metadata (global and index) as needed
clusterService.submitStateUpdateTask("restore_snapshot[" + snapshotName + ']', new ClusterStateUpdateTask() {
final String restoreUUID = UUIDs.randomBase64UUID();
RestoreInfo restoreInfo = null;
@Override
public ClusterState execute(ClusterState currentState) {
RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
// Check if the snapshot to restore is currently being deleted
SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
if (deletionsInProgress != null && deletionsInProgress.getEntries().stream().anyMatch(entry -> entry.getSnapshot().equals(snapshot))) {
throw new ConcurrentSnapshotExecutionException(snapshot, "cannot restore a snapshot while a snapshot deletion is in-progress [" + deletionsInProgress.getEntries().get(0).getSnapshot() + "]");
}
// Updating cluster state
ClusterState.Builder builder = ClusterState.builder(currentState);
Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata());
ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
RoutingTable.Builder rtBuilder = RoutingTable.builder(currentState.routingTable());
ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards;
Set<String> aliases = new HashSet<>();
if (indices.isEmpty() == false) {
// We have some indices to restore
ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> shardsBuilder = ImmutableOpenMap.builder();
final Version minIndexCompatibilityVersion = currentState.getNodes().getMaxNodeVersion().minimumIndexCompatibilityVersion();
for (Map.Entry<String, String> indexEntry : indices.entrySet()) {
String index = indexEntry.getValue();
boolean partial = checkPartial(index);
SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(restoreUUID, snapshot, snapshotInfo.version(), index);
String renamedIndexName = indexEntry.getKey();
IndexMetadata snapshotIndexMetadata = metadata.index(index);
snapshotIndexMetadata = updateIndexSettings(snapshotIndexMetadata, request.indexSettings(), request.ignoreIndexSettings());
try {
snapshotIndexMetadata = metadataIndexUpgradeService.upgradeIndexMetadata(snapshotIndexMetadata, minIndexCompatibilityVersion);
} catch (Exception ex) {
throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex);
}
// Check that the index is closed or doesn't exist
IndexMetadata currentIndexMetadata = currentState.metadata().index(renamedIndexName);
IntSet ignoreShards = new IntHashSet();
final Index renamedIndex;
if (currentIndexMetadata == null) {
// Index doesn't exist - create it and start recovery
// Make sure that the index we are about to create has a validate name
MetadataCreateIndexService.validateIndexName(renamedIndexName, currentState);
createIndexService.validateIndexSettings(renamedIndexName, snapshotIndexMetadata.getSettings(), currentState, false);
IndexMetadata.Builder indexMdBuilder = IndexMetadata.builder(snapshotIndexMetadata).state(IndexMetadata.State.OPEN).index(renamedIndexName);
indexMdBuilder.settings(Settings.builder().put(snapshotIndexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()));
shardLimitValidator.validateShardLimit(snapshotIndexMetadata.getSettings(), currentState);
if (!request.includeAliases() && !snapshotIndexMetadata.getAliases().isEmpty()) {
// Remove all aliases - they shouldn't be restored
indexMdBuilder.removeAllAliases();
} else {
for (ObjectCursor<String> alias : snapshotIndexMetadata.getAliases().keys()) {
aliases.add(alias.value);
}
}
IndexMetadata updatedIndexMetadata = indexMdBuilder.build();
if (partial) {
populateIgnoredShards(index, ignoreShards);
}
rtBuilder.addAsNewRestore(updatedIndexMetadata, recoverySource, ignoreShards);
blocks.addBlocks(updatedIndexMetadata);
mdBuilder.put(updatedIndexMetadata, true);
renamedIndex = updatedIndexMetadata.getIndex();
} else {
validateExistingIndex(currentIndexMetadata, snapshotIndexMetadata, renamedIndexName, partial);
// Index exists and it's closed - open it in metadata and start recovery
IndexMetadata.Builder indexMdBuilder = IndexMetadata.builder(snapshotIndexMetadata).state(IndexMetadata.State.OPEN);
indexMdBuilder.version(Math.max(snapshotIndexMetadata.getVersion(), 1 + currentIndexMetadata.getVersion()));
indexMdBuilder.mappingVersion(Math.max(snapshotIndexMetadata.getMappingVersion(), 1 + currentIndexMetadata.getMappingVersion()));
indexMdBuilder.settingsVersion(Math.max(snapshotIndexMetadata.getSettingsVersion(), 1 + currentIndexMetadata.getSettingsVersion()));
for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) {
indexMdBuilder.primaryTerm(shard, Math.max(snapshotIndexMetadata.primaryTerm(shard), currentIndexMetadata.primaryTerm(shard)));
}
if (!request.includeAliases()) {
// Remove all snapshot aliases
if (!snapshotIndexMetadata.getAliases().isEmpty()) {
indexMdBuilder.removeAllAliases();
}
// Add existing aliases
for (ObjectCursor<AliasMetadata> alias : currentIndexMetadata.getAliases().values()) {
indexMdBuilder.putAlias(alias.value);
}
} else {
for (ObjectCursor<String> alias : snapshotIndexMetadata.getAliases().keys()) {
aliases.add(alias.value);
}
}
indexMdBuilder.settings(Settings.builder().put(snapshotIndexMetadata.getSettings()).put(IndexMetadata.SETTING_INDEX_UUID, currentIndexMetadata.getIndexUUID()));
IndexMetadata updatedIndexMetadata = indexMdBuilder.index(renamedIndexName).build();
rtBuilder.addAsRestore(updatedIndexMetadata, recoverySource);
blocks.updateBlocks(updatedIndexMetadata);
mdBuilder.put(updatedIndexMetadata, true);
renamedIndex = updatedIndexMetadata.getIndex();
}
for (int shard = 0; shard < snapshotIndexMetadata.getNumberOfShards(); shard++) {
if (!ignoreShards.contains(shard)) {
shardsBuilder.put(new ShardId(renamedIndex, shard), new RestoreInProgress.ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId()));
} else {
shardsBuilder.put(new ShardId(renamedIndex, shard), new RestoreInProgress.ShardRestoreStatus(clusterService.state().nodes().getLocalNodeId(), RestoreInProgress.State.FAILURE));
}
}
}
shards = shardsBuilder.build();
RestoreInProgress.Entry restoreEntry = new RestoreInProgress.Entry(restoreUUID, snapshot, overallState(RestoreInProgress.State.INIT, shards), List.copyOf(indices.keySet()), shards);
RestoreInProgress.Builder restoreInProgressBuilder;
if (restoreInProgress != null) {
restoreInProgressBuilder = new RestoreInProgress.Builder(restoreInProgress);
} else {
restoreInProgressBuilder = new RestoreInProgress.Builder();
}
builder.putCustom(RestoreInProgress.TYPE, restoreInProgressBuilder.add(restoreEntry).build());
} else {
shards = ImmutableOpenMap.of();
}
validateExistingTemplates();
checkAliasNameConflicts(indices, aliases);
// Restore templates (but do NOT overwrite existing templates)
restoreTemplates(mdBuilder, currentState);
// Restore global state if needed
if (request.includeGlobalSettings() && metadata.persistentSettings() != null) {
Settings settings = metadata.persistentSettings();
// CrateDB patch to only restore defined settings
if (request.globalSettings().length > 0) {
var filteredSettingBuilder = Settings.builder();
for (String prefix : request.globalSettings()) {
filteredSettingBuilder.put(settings.filter(s -> s.startsWith(prefix)));
}
settings = filteredSettingBuilder.build();
}
clusterSettings.validateUpdate(settings);
mdBuilder.persistentSettings(settings);
}
if (request.includeCustomMetadata() && metadata.customs() != null) {
// CrateDB patch to only restore defined custom metadata types
List<String> customMetadataTypes = Arrays.asList(request.customMetadataTypes());
boolean includeAll = customMetadataTypes.size() == 0;
for (ObjectObjectCursor<String, Metadata.Custom> cursor : metadata.customs()) {
if (!RepositoriesMetadata.TYPE.equals(cursor.key)) {
if (includeAll || customMetadataTypes.contains(cursor.key)) {
mdBuilder.putCustom(cursor.key, cursor.value);
}
}
}
}
if (completed(shards)) {
// We don't have any indices to restore - we are done
restoreInfo = new RestoreInfo(snapshotId.getName(), Collections.unmodifiableList(new ArrayList<>(indices.keySet())), shards.size(), shards.size() - failedShards(shards));
}
RoutingTable rt = rtBuilder.build();
ClusterState updatedState = builder.metadata(mdBuilder).blocks(blocks).routingTable(rt).build();
return allocationService.reroute(updatedState, "restored snapshot [" + snapshot + "]");
}
private void checkAliasNameConflicts(Map<String, String> renamedIndices, Set<String> aliases) {
for (Map.Entry<String, String> renamedIndex : renamedIndices.entrySet()) {
if (aliases.contains(renamedIndex.getKey())) {
throw new SnapshotRestoreException(snapshot, "cannot rename index [" + renamedIndex.getValue() + "] into [" + renamedIndex.getKey() + "] because of conflict with an alias with the same name");
}
}
}
private void populateIgnoredShards(String index, IntSet ignoreShards) {
for (SnapshotShardFailure failure : snapshotInfo.shardFailures()) {
if (index.equals(failure.index())) {
ignoreShards.add(failure.shardId());
}
}
}
private boolean checkPartial(String index) {
// Make sure that index was fully snapshotted
if (failed(snapshotInfo, index)) {
if (request.partial()) {
return true;
} else {
throw new SnapshotRestoreException(snapshot, "index [" + index + "] wasn't fully snapshotted - cannot " + "restore");
}
} else {
return false;
}
}
private void validateExistingIndex(IndexMetadata currentIndexMetadata, IndexMetadata snapshotIndexMetadata, String renamedIndex, boolean partial) {
// Index exist - checking that it's closed
if (currentIndexMetadata.getState() != IndexMetadata.State.CLOSE) {
// TODO: Enable restore for open indices
throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] because an open index " + "with same name already exists in the cluster. Either close or delete the existing index or restore the " + "index under a different name by providing a rename pattern and replacement name");
}
// Index exist - checking if it's partial restore
if (partial) {
throw new SnapshotRestoreException(snapshot, "cannot restore partial index [" + renamedIndex + "] because such index already exists");
}
// Make sure that the number of shards is the same. That's the only thing that we cannot change
if (currentIndexMetadata.getNumberOfShards() != snapshotIndexMetadata.getNumberOfShards()) {
throw new SnapshotRestoreException(snapshot, "cannot restore index [" + renamedIndex + "] with [" + currentIndexMetadata.getNumberOfShards() + "] shards from a snapshot of index [" + snapshotIndexMetadata.getIndex().getName() + "] with [" + snapshotIndexMetadata.getNumberOfShards() + "] shards");
}
}
/**
* Optionally updates index settings in indexMetadata by removing settings listed in ignoreSettings and
* merging them with settings in changeSettings.
*/
private IndexMetadata updateIndexSettings(IndexMetadata indexMetadata, Settings changeSettings, String[] ignoreSettings) {
if (changeSettings.names().isEmpty() && ignoreSettings.length == 0) {
return indexMetadata;
}
Settings normalizedChangeSettings = Settings.builder().put(changeSettings).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build();
IndexMetadata.Builder builder = IndexMetadata.builder(indexMetadata);
Settings settings = indexMetadata.getSettings();
Set<String> keyFilters = new HashSet<>();
List<String> simpleMatchPatterns = new ArrayList<>();
for (String ignoredSetting : ignoreSettings) {
if (!Regex.isSimpleMatchPattern(ignoredSetting)) {
if (UNREMOVABLE_SETTINGS.contains(ignoredSetting)) {
throw new SnapshotRestoreException(snapshot, "cannot remove setting [" + ignoredSetting + "] on restore");
} else {
keyFilters.add(ignoredSetting);
}
} else {
simpleMatchPatterns.add(ignoredSetting);
}
}
Predicate<String> settingsFilter = k -> {
if (UNREMOVABLE_SETTINGS.contains(k) == false) {
for (String filterKey : keyFilters) {
if (k.equals(filterKey)) {
return false;
}
}
for (String pattern : simpleMatchPatterns) {
if (Regex.simpleMatch(pattern, k)) {
return false;
}
}
}
return true;
};
Settings.Builder settingsBuilder = Settings.builder().put(settings.filter(settingsFilter)).put(normalizedChangeSettings.filter(k -> {
if (UNMODIFIABLE_SETTINGS.contains(k)) {
throw new SnapshotRestoreException(snapshot, "cannot modify setting [" + k + "] on restore");
} else {
return true;
}
}));
settingsBuilder.remove(IndexMetadata.VERIFIED_BEFORE_CLOSE_SETTING.getKey());
return builder.settings(settingsBuilder).build();
}
private void restoreTemplates(Metadata.Builder mdBuilder, ClusterState currentState) {
List<String> toRestore = Arrays.asList(request.templates());
if (metadata.templates() != null) {
for (ObjectCursor<IndexTemplateMetadata> cursor : metadata.templates().values()) {
if (currentState.metadata().templates().get(cursor.value.name()) == null && (request.allTemplates() || toRestore.contains(cursor.value.name()))) {
mdBuilder.put(cursor.value);
}
}
}
}
private void validateExistingTemplates() {
if (request.indicesOptions().ignoreUnavailable() || request.allTemplates()) {
return;
}
for (String template : request.templates()) {
if (!metadata.templates().containsKey(template)) {
throw new ResourceNotFoundException("[{}] template not found", template);
}
}
}
@Override
public void onFailure(String source, Exception e) {
LOGGER.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", snapshotId), e);
listener.onFailure(e);
}
@Override
public TimeValue timeout() {
return request.masterNodeTimeout();
}
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
listener.onResponse(new RestoreCompletionResponse(restoreUUID, snapshot, restoreInfo));
}
});
}, listener::onFailure);
}, listener::onFailure);
}));
}, listener::onFailure);
} catch (Exception e) {
LOGGER.warn(() -> new ParameterizedMessage("[{}] failed to restore snapshot", request.repositoryName + ":" + request.snapshotName), e);
listener.onFailure(e);
}
}
use of org.elasticsearch.repositories.Repository in project crate by crate.
the class SysSnapshotsTest method testUnavailableSnapshotsAreFilteredOut.
@Test
public void testUnavailableSnapshotsAreFilteredOut() throws Exception {
HashMap<String, SnapshotId> snapshots = new HashMap<>();
SnapshotId s1 = new SnapshotId("s1", UUIDs.randomBase64UUID());
SnapshotId s2 = new SnapshotId("s2", UUIDs.randomBase64UUID());
snapshots.put(s1.getUUID(), s1);
snapshots.put(s2.getUUID(), s2);
RepositoryData repositoryData = new RepositoryData(1, snapshots, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), ShardGenerations.EMPTY);
Repository r1 = mock(Repository.class);
doAnswer((Answer<Void>) invocation -> {
ActionListener<RepositoryData> callback = invocation.getArgument(0);
callback.onResponse(repositoryData);
return null;
}).when(r1).getRepositoryData(any());
when(r1.getMetadata()).thenReturn(new RepositoryMetadata("repo1", "fs", Settings.EMPTY));
doAnswer((Answer<Void>) invocation -> {
ActionListener<SnapshotInfo> callback = invocation.getArgument(1);
callback.onFailure(new SnapshotException("repo1", "s1", "Everything is wrong"));
return null;
}).when(r1).getSnapshotInfo(eq(s1), any());
doAnswer((Answer<Void>) invocation -> {
ActionListener<SnapshotInfo> callback = invocation.getArgument(1);
callback.onResponse(new SnapshotInfo(s2, Collections.emptyList(), SnapshotState.SUCCESS));
return null;
}).when(r1).getSnapshotInfo(eq(s2), any());
SysSnapshots sysSnapshots = new SysSnapshots(() -> Collections.singletonList(r1));
Stream<SysSnapshot> currentSnapshots = StreamSupport.stream(Spliterators.spliteratorUnknownSize(sysSnapshots.currentSnapshots().get().iterator(), Spliterator.ORDERED), false);
assertThat(currentSnapshots.map(SysSnapshot::name).collect(Collectors.toList()), containsInAnyOrder("s1", "s2"));
}
Aggregations