Examples with RepositoriesService org.opensearch.repositories.RepositoriesService used on opensource projects

Example 6 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class OpenSearchBlobStoreRepositoryIntegTestCase method testIndicesDeletedFromRepository.

public void testIndicesDeletedFromRepository() throws Exception {
    final String repoName = createRepository("test-repo");
    Client client = client();
    createIndex("test-idx-1", "test-idx-2", "test-idx-3");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 20; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    refresh();
    logger.info("--> take a snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, "test-snap").setWaitForCompletion(true).get();
    assertEquals(createSnapshotResponse.getSnapshotInfo().successfulShards(), createSnapshotResponse.getSnapshotInfo().totalShards());
    logger.info("--> indexing more data");
    for (int i = 20; i < 40; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    logger.info("--> take another snapshot with only 2 of the 3 indices");
    createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, "test-snap2").setWaitForCompletion(true).setIndices("test-idx-1", "test-idx-2").get();
    assertEquals(createSnapshotResponse.getSnapshotInfo().successfulShards(), createSnapshotResponse.getSnapshotInfo().totalShards());
    logger.info("--> delete a snapshot");
    assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, "test-snap").get());
    logger.info("--> verify index folder deleted from blob container");
    RepositoriesService repositoriesSvc = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, internalCluster().getMasterName());
    BlobStoreRepository repository = (BlobStoreRepository) repositoriesSvc.repository(repoName);
    final SetOnce<BlobContainer> indicesBlobContainer = new SetOnce<>();
    final PlainActionFuture<RepositoryData> repositoryData = PlainActionFuture.newFuture();
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> {
        indicesBlobContainer.set(repository.blobStore().blobContainer(repository.basePath().add("indices")));
        repository.getRepositoryData(repositoryData);
    });
    for (IndexId indexId : repositoryData.actionGet().getIndices().values()) {
        if (indexId.getName().equals("test-idx-3")) {
            // deleted index
            assertFalse(indicesBlobContainer.get().blobExists(indexId.getId()));
        }
    }
    assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, "test-snap2").get());
}

Example 7 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class OpenSearchMockAPIBasedRepositoryIntegTestCase method testRequestStats.

public void testRequestStats() throws Exception {
    final String repository = createRepository(randomName());
    final String index = "index-no-merges";
    createIndex(index, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build());
    final long nbDocs = randomLongBetween(10_000L, 20_000L);
    try (BackgroundIndexer indexer = new BackgroundIndexer(index, "_doc", client(), (int) nbDocs)) {
        waitForDocs(nbDocs, indexer);
    }
    flushAndRefresh(index);
    ForceMergeResponse forceMerge = client().admin().indices().prepareForceMerge(index).setFlush(true).setMaxNumSegments(1).get();
    assertThat(forceMerge.getSuccessfulShards(), equalTo(1));
    assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
    final String snapshot = "snapshot";
    assertSuccessfulSnapshot(client().admin().cluster().prepareCreateSnapshot(repository, snapshot).setWaitForCompletion(true).setIndices(index));
    assertAcked(client().admin().indices().prepareDelete(index));
    assertSuccessfulRestore(client().admin().cluster().prepareRestoreSnapshot(repository, snapshot).setWaitForCompletion(true));
    ensureGreen(index);
    assertHitCount(client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get(), nbDocs);
    assertAcked(client().admin().cluster().prepareDeleteSnapshot(repository, snapshot).get());
    final RepositoryStats repositoryStats = StreamSupport.stream(internalCluster().getInstances(RepositoriesService.class).spliterator(), false).map(repositoriesService -> {
        try {
            return repositoriesService.repository(repository);
        } catch (RepositoryMissingException e) {
            return null;
        }
    }).filter(Objects::nonNull).map(Repository::stats).reduce(RepositoryStats::merge).get();
    Map<String, Long> sdkRequestCounts = repositoryStats.requestCounts;
    final Map<String, Long> mockCalls = getMockRequestCounts();
    String assertionErrorMsg = String.format("SDK sent [%s] calls and handler measured [%s] calls", sdkRequestCounts, mockCalls);
    assertEquals(assertionErrorMsg, mockCalls, sdkRequestCounts);
}

Example 8 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class AbstractSnapshotIntegTestCase method waitForBlock.

public void waitForBlock(String node, String repository, TimeValue timeout) throws InterruptedException {
    logger.info("--> waiting for [{}] to be blocked on node [{}]", repository, node);
    long start = System.currentTimeMillis();
    RepositoriesService repositoriesService = internalCluster().getInstance(RepositoriesService.class, node);
    MockRepository mockRepository = (MockRepository) repositoriesService.repository(repository);
    while (System.currentTimeMillis() - start < timeout.millis()) {
        if (mockRepository.blocked()) {
            return;
        }
        Thread.sleep(100);
    }
    fail("Timeout waiting for node [" + node + "] to be blocked");
}

Example 9 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class AbstractSnapshotIntegTestCase method waitForBlockOnAnyDataNode.

public static void waitForBlockOnAnyDataNode(String repository, TimeValue timeout) throws InterruptedException {
    final boolean blocked = waitUntil(() -> {
        for (RepositoriesService repositoriesService : internalCluster().getDataNodeInstances(RepositoriesService.class)) {
            MockRepository mockRepository = (MockRepository) repositoriesService.repository(repository);
            if (mockRepository.blocked()) {
                return true;
            }
        }
        return false;
    }, timeout.millis(), TimeUnit.MILLISECONDS);
    assertTrue("No repository is blocked waiting on a data node", blocked);
}

Example 10 with RepositoriesService

use of org.opensearch.repositories.RepositoriesService in project OpenSearch by opensearch-project.

the class BlobStoreRepositoryTests method testRetrieveSnapshots.

public void testRetrieveSnapshots() throws Exception {
    final Client client = client();
    final Path location = OpenSearchIntegTestCase.randomRepoPath(node().settings());
    final String repositoryName = "test-repo";
    logger.info("-->  creating repository");
    AcknowledgedResponse putRepositoryResponse = client.admin().cluster().preparePutRepository(repositoryName).setType(REPO_TYPE).setSettings(Settings.builder().put(node().settings()).put("location", location)).get();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    logger.info("--> creating an index and indexing documents");
    final String indexName = "test-idx";
    createIndex(indexName);
    ensureGreen();
    int numDocs = randomIntBetween(10, 20);
    for (int i = 0; i < numDocs; i++) {
        String id = Integer.toString(i);
        client().prepareIndex(indexName).setId(id).setSource("text", "sometext").get();
    }
    client().admin().indices().prepareFlush(indexName).get();
    logger.info("--> create first snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, "test-snap-1").setWaitForCompletion(true).setIndices(indexName).get();
    final SnapshotId snapshotId1 = createSnapshotResponse.getSnapshotInfo().snapshotId();
    logger.info("--> create second snapshot");
    createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, "test-snap-2").setWaitForCompletion(true).setIndices(indexName).get();
    final SnapshotId snapshotId2 = createSnapshotResponse.getSnapshotInfo().snapshotId();
    logger.info("--> make sure the node's repository can resolve the snapshots");
    final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
    final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repositoryName);
    final List<SnapshotId> originalSnapshots = Arrays.asList(snapshotId1, snapshotId2);
    List<SnapshotId> snapshotIds = OpenSearchBlobStoreRepositoryIntegTestCase.getRepositoryData(repository).getSnapshotIds().stream().sorted((s1, s2) -> s1.getName().compareTo(s2.getName())).collect(Collectors.toList());
    assertThat(snapshotIds, equalTo(originalSnapshots));
}