Examples with RepositoriesService org.opensearch.repositories.RepositoriesService used on opensource projects

Example 1 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryCleanupIT method testCleanupOldIndexN.

public void testCleanupOldIndexN() throws ExecutionException, InterruptedException {
    internalCluster().startNodes(Settings.EMPTY);
    final String repoName = "test-repo";
    createRepository(repoName, "fs");
    logger.info("--> create three snapshots");
    for (int i = 0; i < 3; ++i) {
        CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot(repoName, "test-snap-" + i).setWaitForCompletion(true).get();
        assertThat(createSnapshotResponse.getSnapshotInfo().state(), is(SnapshotState.SUCCESS));
    }
    final RepositoriesService service = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    final BlobStoreRepository repository = (BlobStoreRepository) service.repository(repoName);
    logger.info("--> write two outdated index-N blobs");
    for (int i = 0; i < 2; ++i) {
        final PlainActionFuture<Void> createOldIndexNFuture = PlainActionFuture.newFuture();
        final int generation = i;
        repository.threadPool().generic().execute(ActionRunnable.run(createOldIndexNFuture, () -> repository.blobStore().blobContainer(repository.basePath()).writeBlob(BlobStoreRepository.INDEX_FILE_PREFIX + generation, new ByteArrayInputStream(new byte[1]), 1, true)));
        createOldIndexNFuture.get();
    }
    logger.info("--> cleanup repository");
    client().admin().cluster().prepareCleanupRepository(repoName).get();
    BlobStoreTestUtil.assertConsistency(repository, repository.threadPool().generic());
}

Example 2 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryCleanupIT method startBlockedCleanup.

private String startBlockedCleanup(String repoName) throws Exception {
    logger.info("-->  starting two master nodes and one data node");
    internalCluster().startMasterOnlyNodes(2);
    internalCluster().startDataOnlyNodes(1);
    createRepository(repoName, "mock");
    logger.info("-->  snapshot");
    client().admin().cluster().prepareCreateSnapshot(repoName, "test-snap").setWaitForCompletion(true).get();
    final RepositoriesService service = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    final BlobStoreRepository repository = (BlobStoreRepository) service.repository(repoName);
    logger.info("--> creating a garbage data blob");
    final PlainActionFuture<Void> garbageFuture = PlainActionFuture.newFuture();
    repository.threadPool().generic().execute(ActionRunnable.run(garbageFuture, () -> repository.blobStore().blobContainer(repository.basePath()).writeBlob("snap-foo.dat", new ByteArrayInputStream(new byte[1]), 1, true)));
    garbageFuture.get();
    final String masterNode = blockMasterFromFinalizingSnapshotOnIndexFile(repoName);
    logger.info("--> starting repository cleanup");
    client().admin().cluster().prepareCleanupRepository(repoName).execute();
    logger.info("--> waiting for block to kick in on " + masterNode);
    waitForBlock(masterNode, repoName, TimeValue.timeValueSeconds(60));
    awaitClusterState(state -> state.custom(RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress.EMPTY).hasCleanupInProgress());
    return masterNode;
}

Example 3 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class RestoreSnapshotIT method testDynamicRestoreThrottling.

public void testDynamicRestoreThrottling() throws Exception {
    Client client = client();
    createRepository("test-repo", "fs", Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()).put("chunk_size", 100, ByteSizeUnit.BYTES));
    createIndexWithRandomDocs("test-idx", 100);
    logger.info("--> snapshot");
    client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
    logger.info("--> delete index");
    cluster().wipeIndices("test-idx");
    logger.info("--> restore index");
    client.admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), "100b").build()).get();
    ActionFuture<RestoreSnapshotResponse> restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute();
    // check if throttling is active
    assertBusy(() -> {
        long restorePause = 0L;
        for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) {
            restorePause += repositoriesService.repository("test-repo").getRestoreThrottleTimeInNanos();
        }
        assertThat(restorePause, greaterThan(TimeValue.timeValueSeconds(randomIntBetween(1, 5)).nanos()));
        assertFalse(restoreSnapshotResponse.isDone());
    }, 30, TimeUnit.SECONDS);
    // run at full speed again
    client.admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().putNull(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()).build()).get();
    // check that restore now completes quickly (i.e. within 20 seconds)
    assertThat(restoreSnapshotResponse.get(20L, TimeUnit.SECONDS).getRestoreInfo().totalShards(), greaterThan(0));
    assertDocCount("test-idx", 100L);
}

Example 4 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class IndexShard method startRecovery.

public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService, PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService, Consumer<MappingMetadata> mappingUpdateConsumer, IndicesService indicesService) {
    // }
    assert recoveryState.getRecoverySource().equals(shardRouting.recoverySource());
    switch(recoveryState.getRecoverySource().getType()) {
        case EMPTY_STORE:
        case EXISTING_STORE:
            executeRecovery("from store", recoveryState, recoveryListener, this::recoverFromStore);
            break;
        case PEER:
            try {
                markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState);
                recoveryTargetService.startRecovery(this, recoveryState.getSourceNode(), recoveryListener);
            } catch (Exception e) {
                failShard("corrupted preexisting index", e);
                recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
            }
            break;
        case SNAPSHOT:
            final String repo = ((SnapshotRecoverySource) recoveryState.getRecoverySource()).snapshot().getRepository();
            executeRecovery("from snapshot", recoveryState, recoveryListener, l -> restoreFromRepository(repositoriesService.repository(repo), l));
            break;
        case LOCAL_SHARDS:
            final IndexMetadata indexMetadata = indexSettings().getIndexMetadata();
            final Index resizeSourceIndex = indexMetadata.getResizeSourceIndex();
            final List<IndexShard> startedShards = new ArrayList<>();
            final IndexService sourceIndexService = indicesService.indexService(resizeSourceIndex);
            final Set<ShardId> requiredShards;
            final int numShards;
            if (sourceIndexService != null) {
                requiredShards = IndexMetadata.selectRecoverFromShards(shardId().id(), sourceIndexService.getMetadata(), indexMetadata.getNumberOfShards());
                for (IndexShard shard : sourceIndexService) {
                    if (shard.state() == IndexShardState.STARTED && requiredShards.contains(shard.shardId())) {
                        startedShards.add(shard);
                    }
                }
                numShards = requiredShards.size();
            } else {
                numShards = -1;
                requiredShards = Collections.emptySet();
            }
            if (numShards == startedShards.size()) {
                assert requiredShards.isEmpty() == false;
                executeRecovery("from local shards", recoveryState, recoveryListener, l -> recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream().filter((s) -> requiredShards.contains(s.shardId())).collect(Collectors.toList()), l));
            } else {
                final RuntimeException e;
                if (numShards == -1) {
                    e = new IndexNotFoundException(resizeSourceIndex);
                } else {
                    e = new IllegalStateException("not all required shards of index " + resizeSourceIndex + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId());
                }
                throw e;
            }
            break;
        default:
            throw new IllegalArgumentException("Unknown recovery source " + recoveryState.getRecoverySource());
    }
}

Example 5 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryTests method testRetrieveSnapshots.

public void testRetrieveSnapshots() throws Exception {
    final Client client = client();
    final Path location = OpenSearchIntegTestCase.randomRepoPath(node().settings());
    final String repositoryName = "test-repo";
    logger.info("-->  creating repository");
    AcknowledgedResponse putRepositoryResponse = client.admin().cluster().preparePutRepository(repositoryName).setType(REPO_TYPE).setSettings(Settings.builder().put(node().settings()).put("location", location)).get();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    logger.info("--> creating an index and indexing documents");
    final String indexName = "test-idx";
    createIndex(indexName);
    ensureGreen();
    int numDocs = randomIntBetween(10, 20);
    for (int i = 0; i < numDocs; i++) {
        String id = Integer.toString(i);
        client().prepareIndex(indexName).setId(id).setSource("text", "sometext").get();
    }
    client().admin().indices().prepareFlush(indexName).get();
    logger.info("--> create first snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, "test-snap-1").setWaitForCompletion(true).setIndices(indexName).get();
    final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId();
    logger.info("--> create second snapshot");
    createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, "test-snap-2").setWaitForCompletion(true).setIndices(indexName).get();
    final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId();
    logger.info("--> make sure the node's repository can resolve the snapshots");
    final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
    final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repositoryName);
    final List<SnapshotId> originalSnapshots = Arrays.asList(snapshotId1, snapshotId2);
    List<SnapshotId> snapshotIds = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository).getSnapshotIds().stream().sorted((s1, s2) -> s1.getName().compareTo(s2.getName())).collect(Collectors.toList());
    assertThat(snapshotIds, equalTo(originalSnapshots));
}