Examples with SnapshotInfo org.opensearch.snapshots.SnapshotInfo used on opensource projects

Example 11 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryRestoreTests method testSnapshotWithConflictingName.

public void testSnapshotWithConflictingName() throws Exception {
    final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
    final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0);
    IndexShard shard = newShard(shardId, true);
    try {
        // index documents in the shards
        final int numDocs = scaledRandomIntBetween(1, 500);
        recoverShardFromStore(shard);
        for (int i = 0; i < numDocs; i++) {
            indexDoc(shard, "_doc", Integer.toString(i));
            if (rarely()) {
                flushShard(shard, false);
            }
        }
        assertDocCount(shard, numDocs);
        // snapshot the shard
        final Repository repository = createRepository();
        final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid"));
        final String shardGen = snapshotShard(shard, snapshot, repository);
        assertNotNull(shardGen);
        final Snapshot snapshotWithSameName = new Snapshot(repository.getMetadata().name(), new SnapshotId(snapshot.getSnapshotId().getName(), "_uuid2"));
        final ShardGenerations shardGenerations = ShardGenerations.builder().put(indexId, 0, shardGen).build();
        PlainActionFuture.<RepositoryData, Exception>get(f -> repository.finalizeSnapshot(shardGenerations, RepositoryData.EMPTY_REPO_GEN, Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), new SnapshotInfo(snapshot.getSnapshotId(), shardGenerations.indices().stream().map(IndexId::getName).collect(Collectors.toList()), Collections.emptyList(), 0L, null, 1L, 6, Collections.emptyList(), true, Collections.emptyMap()), Version.CURRENT, Function.identity(), f));
        IndexShardSnapshotFailedException isfe = expectThrows(IndexShardSnapshotFailedException.class, () -> snapshotShard(shard, snapshotWithSameName, repository));
        assertThat(isfe.getMessage(), containsString("Duplicate snapshot name"));
    } finally {
        if (shard != null && shard.state() != IndexShardState.CLOSED) {
            try {
                shard.close("test", false);
            } finally {
                IOUtils.close(shard.store());
            }
        }
    }
}

Example 12 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class BlobStoreRepository method finalizeSnapshot.

@Override
public void finalizeSnapshot(final ShardGenerations shardGenerations, final long repositoryStateId, final Metadata clusterMetadata, SnapshotInfo snapshotInfo, Version repositoryMetaVersion, Function<ClusterState, ClusterState> stateTransformer, final ActionListener<RepositoryData> listener) {
    assert repositoryStateId > RepositoryData.UNKNOWN_REPO_GEN : "Must finalize based on a valid repository generation but received [" + repositoryStateId + "]";
    final Collection<IndexId> indices = shardGenerations.indices();
    final SnapshotId snapshotId = snapshotInfo.snapshotId();
    // Once we are done writing the updated index-N blob we remove the now unreferenced index-${uuid} blobs in each shard
    // directory if all nodes are at least at version SnapshotsService#SHARD_GEN_IN_REPO_DATA_VERSION
    // If there are older version nodes in the cluster, we don't need to run this cleanup as it will have already happened
    // when writing the index-${N} to each shard directory.
    final boolean writeShardGens = SnapshotsService.useShardGenerations(repositoryMetaVersion);
    final Consumer<Exception> onUpdateFailure = e -> listener.onFailure(new SnapshotException(metadata.name(), snapshotId, "failed to update snapshot in repository", e));
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    final boolean writeIndexGens = SnapshotsService.useIndexGenerations(repositoryMetaVersion);
    final StepListener<RepositoryData> repoDataListener = new StepListener<>();
    getRepositoryData(repoDataListener);
    repoDataListener.whenComplete(existingRepositoryData -> {
        final Map<IndexId, String> indexMetas;
        final Map<String, String> indexMetaIdentifiers;
        if (writeIndexGens) {
            indexMetaIdentifiers = ConcurrentCollections.newConcurrentMap();
            indexMetas = ConcurrentCollections.newConcurrentMap();
        } else {
            indexMetas = null;
            indexMetaIdentifiers = null;
        }
        final ActionListener<Void> allMetaListener = new GroupedActionListener<>(ActionListener.wrap(v -> {
            final RepositoryData updatedRepositoryData = existingRepositoryData.addSnapshot(snapshotId, snapshotInfo.state(), Version.CURRENT, shardGenerations, indexMetas, indexMetaIdentifiers);
            writeIndexGen(updatedRepositoryData, repositoryStateId, repositoryMetaVersion, stateTransformer, ActionListener.wrap(newRepoData -> {
                if (writeShardGens) {
                    cleanupOldShardGens(existingRepositoryData, updatedRepositoryData);
                }
                listener.onResponse(newRepoData);
            }, onUpdateFailure));
        }, onUpdateFailure), 2 + indices.size());
        // We ignore all FileAlreadyExistsException when writing metadata since otherwise a cluster-manager failover
        // while in this method will mean that no snap-${uuid}.dat blob is ever written for this snapshot. This is safe because
        // any updated version of the index or global metadata will be compatible with the segments written in this snapshot as well.
        // Failing on an already existing index-${repoGeneration} below ensures that the index.latest blob is not updated in a way
        // that decrements the generation it points at
        // Write Global MetaData
        executor.execute(ActionRunnable.run(allMetaListener, () -> GLOBAL_METADATA_FORMAT.write(clusterMetadata, blobContainer(), snapshotId.getUUID(), compress)));
        // write the index metadata for each index in the snapshot
        for (IndexId index : indices) {
            executor.execute(ActionRunnable.run(allMetaListener, () -> {
                final IndexMetadata indexMetaData = clusterMetadata.index(index.getName());
                if (writeIndexGens) {
                    final String identifiers = IndexMetaDataGenerations.buildUniqueIdentifier(indexMetaData);
                    String metaUUID = existingRepositoryData.indexMetaDataGenerations().getIndexMetaBlobId(identifiers);
                    if (metaUUID == null) {
                        // We don't yet have this version of the metadata so we write it
                        metaUUID = UUIDs.base64UUID();
                        INDEX_METADATA_FORMAT.write(indexMetaData, indexContainer(index), metaUUID, compress);
                        indexMetaIdentifiers.put(identifiers, metaUUID);
                    }
                    indexMetas.put(index, identifiers);
                } else {
                    INDEX_METADATA_FORMAT.write(clusterMetadata.index(index.getName()), indexContainer(index), snapshotId.getUUID(), compress);
                }
            }));
        }
        executor.execute(ActionRunnable.run(allMetaListener, () -> SNAPSHOT_FORMAT.write(snapshotInfo, blobContainer(), snapshotId.getUUID(), compress)));
    }, onUpdateFailure);
}

Example 13 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class SnapshotDisruptionIT method testDisruptionAfterShardFinalization.

public void testDisruptionAfterShardFinalization() throws Exception {
    final String idxName = "test";
    internalCluster().startMasterOnlyNodes(1);
    internalCluster().startDataOnlyNode();
    ensureStableCluster(2);
    createIndex(idxName);
    index(idxName, "type", JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject());
    final String repoName = "test-repo";
    createRepository(repoName, "mock");
    final String clusterManagerNode = internalCluster().getMasterName();
    blockAllDataNodes(repoName);
    final String snapshot = "test-snap";
    logger.info("--> starting snapshot");
    ActionFuture<CreateSnapshotResponse> future = client(clusterManagerNode).admin().cluster().prepareCreateSnapshot(repoName, snapshot).setWaitForCompletion(true).execute();
    waitForBlockOnAnyDataNode(repoName, TimeValue.timeValueSeconds(10L));
    NetworkDisruption networkDisruption = isolateClusterManagerDisruption(NetworkDisruption.DISCONNECT);
    internalCluster().setDisruptionScheme(networkDisruption);
    networkDisruption.startDisrupting();
    final CreateSnapshotResponse createSnapshotResponse = future.get();
    final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
    assertThat(snapshotInfo.state(), is(SnapshotState.PARTIAL));
    logger.info("--> stopping disrupting");
    networkDisruption.stopDisrupting();
    unblockAllDataNodes(repoName);
    ensureStableCluster(2, clusterManagerNode);
    logger.info("--> done");
    logger.info("--> recreate the index with potentially different shard counts");
    client().admin().indices().prepareDelete(idxName).get();
    createIndex(idxName);
    index(idxName, "type", JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject());
    logger.info("--> run a snapshot that fails to finalize but succeeds on the data node");
    blockMasterFromFinalizingSnapshotOnIndexFile(repoName);
    final ActionFuture<CreateSnapshotResponse> snapshotFuture = client(clusterManagerNode).admin().cluster().prepareCreateSnapshot(repoName, "snapshot-2").setWaitForCompletion(true).execute();
    waitForBlock(clusterManagerNode, repoName, TimeValue.timeValueSeconds(10L));
    unblockNode(repoName, clusterManagerNode);
    assertFutureThrows(snapshotFuture, SnapshotException.class);
    logger.info("--> create a snapshot expected to be successful");
    final CreateSnapshotResponse successfulSnapshot = client(clusterManagerNode).admin().cluster().prepareCreateSnapshot(repoName, "snapshot-2").setWaitForCompletion(true).get();
    final SnapshotInfo successfulSnapshotInfo = successfulSnapshot.getSnapshotInfo();
    assertThat(successfulSnapshotInfo.state(), is(SnapshotState.SUCCESS));
    logger.info("--> making sure snapshot delete works out cleanly");
    assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, "snapshot-2").get());
}

Example 14 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class DiskThresholdDeciderIT method testRestoreSnapshotAllocationDoesNotExceedWatermark.

public void testRestoreSnapshotAllocationDoesNotExceedWatermark() throws Exception {
    internalCluster().startClusterManagerOnlyNode();
    internalCluster().startDataOnlyNode();
    final String dataNodeName = internalCluster().startDataOnlyNode();
    ensureStableCluster(3);
    assertAcked(client().admin().cluster().preparePutRepository("repo").setType(FsRepository.TYPE).setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean())));
    final InternalClusterInfoService clusterInfoService = (InternalClusterInfoService) internalCluster().getCurrentMasterNodeInstance(ClusterInfoService.class);
    internalCluster().getCurrentMasterNodeInstance(ClusterService.class).addListener(event -> clusterInfoService.refresh());
    final String dataNode0Id = internalCluster().getInstance(NodeEnvironment.class, dataNodeName).nodeId();
    final Path dataNode0Path = internalCluster().getInstance(Environment.class, dataNodeName).dataFiles()[0];
    final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
    createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 6).put(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING.getKey(), "0ms").build());
    final long minShardSize = createReasonableSizedShards(indexName);
    final CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("repo", "snap").setWaitForCompletion(true).get();
    final SnapshotInfo snapshotInfo = createSnapshotResponse.getSnapshotInfo();
    assertThat(snapshotInfo.successfulShards(), is(snapshotInfo.totalShards()));
    assertThat(snapshotInfo.state(), is(SnapshotState.SUCCESS));
    assertAcked(client().admin().indices().prepareDelete(indexName).get());
    // reduce disk size of node 0 so that no shards fit below the low watermark, forcing shards to be assigned to the other data node
    fileSystemProvider.getTestFileStore(dataNode0Path).setTotalSpace(minShardSize + WATERMARK_BYTES - 1L);
    refreshDiskUsage();
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString()).build()).get());
    final RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("repo", "snap").setWaitForCompletion(true).get();
    final RestoreInfo restoreInfo = restoreSnapshotResponse.getRestoreInfo();
    assertThat(restoreInfo.successfulShards(), is(snapshotInfo.totalShards()));
    assertThat(restoreInfo.failedShards(), is(0));
    assertBusy(() -> assertThat(getShardRoutings(dataNode0Id, indexName), empty()));
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().putNull(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey()).build()).get());
    // increase disk size of node 0 to allow just enough room for one shard, and check that it's rebalanced back
    fileSystemProvider.getTestFileStore(dataNode0Path).setTotalSpace(minShardSize + WATERMARK_BYTES + 1L);
    assertBusyWithDiskUsageRefresh(dataNode0Id, indexName, hasSize(1));
}

Example 15 with SnapshotInfo

use of org.opensearch.snapshots.SnapshotInfo in project OpenSearch by opensearch-project.

the class RestSnapshotAction method buildTable.

private Table buildTable(RestRequest req, GetSnapshotsResponse getSnapshotsResponse) {
    Table table = getTableWithHeader(req);
    for (SnapshotInfo snapshotStatus : getSnapshotsResponse.getSnapshots()) {
        table.startRow();
        table.addCell(snapshotStatus.snapshotId().getName());
        table.addCell(snapshotStatus.state());
        table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS));
        table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.startTime())));
        table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.endTime(), TimeUnit.MILLISECONDS));
        table.addCell(FORMATTER.format(Instant.ofEpochMilli(snapshotStatus.endTime())));
        final long durationMillis;
        if (snapshotStatus.state() == SnapshotState.IN_PROGRESS) {
            durationMillis = System.currentTimeMillis() - snapshotStatus.startTime();
        } else {
            durationMillis = snapshotStatus.endTime() - snapshotStatus.startTime();
        }
        table.addCell(TimeValue.timeValueMillis(durationMillis));
        table.addCell(snapshotStatus.indices().size());
        table.addCell(snapshotStatus.successfulShards());
        table.addCell(snapshotStatus.failedShards());
        table.addCell(snapshotStatus.totalShards());
        table.addCell(snapshotStatus.reason());
        table.endRow();
    }
    return table;
}