Examples with FilterRepository org.opensearch.repositories.FilterRepository used on opensource projects

Example 1 with FilterRepository

use of org.opensearch.repositories.FilterRepository in project OpenSearch by opensearch-project.

the class InternalSnapshotsInfoServiceTests method testNoLongerMaster.

public void testNoLongerMaster() throws Exception {
    final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService(Settings.EMPTY, clusterService, () -> repositoriesService, () -> rerouteService);
    final Repository mockRepository = new FilterRepository(mock(Repository.class)) {

        @Override
        public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) {
            return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, randomNonNegativeLong(), null);
        }
    };
    when(repositoriesService.repository("_repo")).thenReturn(mockRepository);
    for (int i = 0; i < randomIntBetween(1, 10); i++) {
        final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
        final int nbShards = randomIntBetween(1, 5);
        applyClusterState("restore-indices-when-master-" + indexName, clusterState -> addUnassignedShards(clusterState, indexName, nbShards));
    }
    applyClusterState("demote-current-master", this::demoteMasterNode);
    for (int i = 0; i < randomIntBetween(1, 10); i++) {
        final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
        final int nbShards = randomIntBetween(1, 5);
        applyClusterState("restore-indices-when-no-longer-master-" + indexName, clusterState -> addUnassignedShards(clusterState, indexName, nbShards));
    }
    assertBusy(() -> {
        assertThat(snapshotsInfoService.numberOfKnownSnapshotShardSizes(), equalTo(0));
        assertThat(snapshotsInfoService.numberOfUnknownSnapshotShardSizes(), equalTo(0));
        assertThat(snapshotsInfoService.numberOfFailedSnapshotShardSizes(), equalTo(0));
    });
}

Example 2 with FilterRepository

use of org.opensearch.repositories.FilterRepository in project OpenSearch by opensearch-project.

the class InternalSnapshotsInfoServiceTests method testCleanUpSnapshotShardSizes.

public void testCleanUpSnapshotShardSizes() throws Exception {
    final Repository mockRepository = new FilterRepository(mock(Repository.class)) {

        @Override
        public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) {
            if (randomBoolean()) {
                throw new SnapshotException(new Snapshot("_repo", snapshotId), "simulated");
            } else {
                return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, randomNonNegativeLong(), null);
            }
        }
    };
    when(repositoriesService.repository("_repo")).thenReturn(mockRepository);
    final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService(Settings.EMPTY, clusterService, () -> repositoriesService, () -> rerouteService);
    final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
    final int nbShards = randomIntBetween(1, 10);
    applyClusterState("new snapshot restore for index " + indexName, clusterState -> addUnassignedShards(clusterState, indexName, nbShards));
    // waiting for snapshot shard size fetches to be executed, as we want to verify that they are cleaned up
    assertBusy(() -> assertThat(snapshotsInfoService.numberOfFailedSnapshotShardSizes() + snapshotsInfoService.numberOfKnownSnapshotShardSizes(), equalTo(nbShards)));
    if (randomBoolean()) {
        // simulate initialization and start of the shards
        final AllocationService allocationService = OpenSearchAllocationTestCase.createAllocationService(Settings.builder().put(CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_RECOVERIES_SETTING.getKey(), nbShards).put(CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING.getKey(), nbShards).build(), snapshotsInfoService);
        applyClusterState("starting shards for " + indexName, clusterState -> OpenSearchAllocationTestCase.startInitializingShardsAndReroute(allocationService, clusterState, indexName));
        assertTrue(clusterService.state().routingTable().shardsWithState(ShardRoutingState.UNASSIGNED).isEmpty());
    } else {
        // simulate deletion of the index
        applyClusterState("delete index " + indexName, clusterState -> deleteIndex(clusterState, indexName));
        assertFalse(clusterService.state().metadata().hasIndex(indexName));
    }
    assertThat(snapshotsInfoService.numberOfKnownSnapshotShardSizes(), equalTo(0));
    assertThat(snapshotsInfoService.numberOfUnknownSnapshotShardSizes(), equalTo(0));
    assertThat(snapshotsInfoService.numberOfFailedSnapshotShardSizes(), equalTo(0));
}

Example 3 with FilterRepository

use of org.opensearch.repositories.FilterRepository in project OpenSearch by opensearch-project.

the class InternalSnapshotsInfoServiceTests method testSnapshotShardSizes.

public void testSnapshotShardSizes() throws Exception {
    final int maxConcurrentFetches = randomIntBetween(1, 10);
    final int numberOfShards = randomIntBetween(1, 50);
    final CountDownLatch rerouteLatch = new CountDownLatch(numberOfShards);
    final RerouteService rerouteService = (reason, priority, listener) -> {
        listener.onResponse(clusterService.state());
        assertThat(rerouteLatch.getCount(), greaterThanOrEqualTo(0L));
        rerouteLatch.countDown();
    };
    final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService(Settings.builder().put(INTERNAL_SNAPSHOT_INFO_MAX_CONCURRENT_FETCHES_SETTING.getKey(), maxConcurrentFetches).build(), clusterService, () -> repositoriesService, () -> rerouteService);
    final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
    final long[] expectedShardSizes = new long[numberOfShards];
    for (int i = 0; i < expectedShardSizes.length; i++) {
        expectedShardSizes[i] = randomNonNegativeLong();
    }
    final AtomicInteger getShardSnapshotStatusCount = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(1);
    final Repository mockRepository = new FilterRepository(mock(Repository.class)) {

        @Override
        public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) {
            try {
                assertThat(indexId.getName(), equalTo(indexName));
                assertThat(shardId.id(), allOf(greaterThanOrEqualTo(0), lessThan(numberOfShards)));
                latch.await();
                getShardSnapshotStatusCount.incrementAndGet();
                return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, expectedShardSizes[shardId.id()], null);
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
        }
    };
    when(repositoriesService.repository("_repo")).thenReturn(mockRepository);
    applyClusterState("add-unassigned-shards", clusterState -> addUnassignedShards(clusterState, indexName, numberOfShards));
    waitForMaxActiveGenericThreads(Math.min(numberOfShards, maxConcurrentFetches));
    if (randomBoolean()) {
        applyClusterState("reapply-last-cluster-state-to-check-deduplication-works", state -> ClusterState.builder(state).incrementVersion().build());
    }
    assertThat(snapshotsInfoService.numberOfUnknownSnapshotShardSizes(), equalTo(numberOfShards));
    assertThat(snapshotsInfoService.numberOfKnownSnapshotShardSizes(), equalTo(0));
    latch.countDown();
    assertTrue(rerouteLatch.await(30L, TimeUnit.SECONDS));
    assertThat(snapshotsInfoService.numberOfKnownSnapshotShardSizes(), equalTo(numberOfShards));
    assertThat(snapshotsInfoService.numberOfUnknownSnapshotShardSizes(), equalTo(0));
    assertThat(snapshotsInfoService.numberOfFailedSnapshotShardSizes(), equalTo(0));
    assertThat(getShardSnapshotStatusCount.get(), equalTo(numberOfShards));
    final SnapshotShardSizeInfo snapshotShardSizeInfo = snapshotsInfoService.snapshotShardSizes();
    for (int i = 0; i < numberOfShards; i++) {
        final ShardRouting shardRouting = clusterService.state().routingTable().index(indexName).shard(i).primaryShard();
        assertThat(snapshotShardSizeInfo.getShardSize(shardRouting), equalTo(expectedShardSizes[i]));
        assertThat(snapshotShardSizeInfo.getShardSize(shardRouting, Long.MIN_VALUE), equalTo(expectedShardSizes[i]));
    }
}

Example 4 with FilterRepository

use of org.opensearch.repositories.FilterRepository in project OpenSearch by opensearch-project.

the class InternalSnapshotsInfoServiceTests method testErroneousSnapshotShardSizes.

public void testErroneousSnapshotShardSizes() throws Exception {
    final AtomicInteger reroutes = new AtomicInteger();
    final RerouteService rerouteService = (reason, priority, listener) -> {
        reroutes.incrementAndGet();
        listener.onResponse(clusterService.state());
    };
    final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService(Settings.builder().put(INTERNAL_SNAPSHOT_INFO_MAX_CONCURRENT_FETCHES_SETTING.getKey(), randomIntBetween(1, 10)).build(), clusterService, () -> repositoriesService, () -> rerouteService);
    final Map<InternalSnapshotsInfoService.SnapshotShard, Long> results = new ConcurrentHashMap<>();
    final Repository mockRepository = new FilterRepository(mock(Repository.class)) {

        @Override
        public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) {
            final InternalSnapshotsInfoService.SnapshotShard snapshotShard = new InternalSnapshotsInfoService.SnapshotShard(new Snapshot("_repo", snapshotId), indexId, shardId);
            if (randomBoolean()) {
                results.put(snapshotShard, Long.MIN_VALUE);
                throw new SnapshotException(snapshotShard.snapshot(), "simulated");
            } else {
                final long shardSize = randomNonNegativeLong();
                results.put(snapshotShard, shardSize);
                return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, shardSize, null);
            }
        }
    };
    when(repositoriesService.repository("_repo")).thenReturn(mockRepository);
    final int maxShardsToCreate = scaledRandomIntBetween(10, 500);
    final Thread addSnapshotRestoreIndicesThread = new Thread(() -> {
        int remainingShards = maxShardsToCreate;
        while (remainingShards > 0) {
            final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
            final int numberOfShards = randomIntBetween(1, remainingShards);
            try {
                applyClusterState("add-more-unassigned-shards-for-" + indexName, clusterState -> addUnassignedShards(clusterState, indexName, numberOfShards));
            } catch (Exception e) {
                throw new AssertionError(e);
            } finally {
                remainingShards -= numberOfShards;
            }
        }
    });
    addSnapshotRestoreIndicesThread.start();
    addSnapshotRestoreIndicesThread.join();
    final Predicate<Long> failedSnapshotShardSizeRetrieval = shardSize -> shardSize == Long.MIN_VALUE;
    assertBusy(() -> {
        assertThat(snapshotsInfoService.numberOfKnownSnapshotShardSizes(), equalTo((int) results.values().stream().filter(size -> failedSnapshotShardSizeRetrieval.test(size) == false).count()));
        assertThat(snapshotsInfoService.numberOfFailedSnapshotShardSizes(), equalTo((int) results.values().stream().filter(failedSnapshotShardSizeRetrieval).count()));
        assertThat(snapshotsInfoService.numberOfUnknownSnapshotShardSizes(), equalTo(0));
    });
    final SnapshotShardSizeInfo snapshotShardSizeInfo = snapshotsInfoService.snapshotShardSizes();
    for (Map.Entry<InternalSnapshotsInfoService.SnapshotShard, Long> snapshotShard : results.entrySet()) {
        final ShardId shardId = snapshotShard.getKey().shardId();
        final ShardRouting shardRouting = clusterService.state().routingTable().index(shardId.getIndexName()).shard(shardId.id()).primaryShard();
        assertThat(shardRouting, notNullValue());
        final boolean success = failedSnapshotShardSizeRetrieval.test(snapshotShard.getValue()) == false;
        assertThat(snapshotShardSizeInfo.getShardSize(shardRouting), success ? equalTo(results.get(snapshotShard.getKey())) : equalTo(ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE));
        final long defaultValue = randomNonNegativeLong();
        assertThat(snapshotShardSizeInfo.getShardSize(shardRouting, defaultValue), success ? equalTo(results.get(snapshotShard.getKey())) : equalTo(defaultValue));
    }
    assertThat("Expecting all snapshot shard size fetches to provide a size", results.size(), equalTo(maxShardsToCreate));
    assertThat("Expecting all snapshot shard size fetches to execute a Reroute", reroutes.get(), equalTo(maxShardsToCreate));
}

Example 5 with FilterRepository

use of org.opensearch.repositories.FilterRepository in project OpenSearch by opensearch-project.

the class InternalSnapshotsInfoServiceTests method testNoLongerClusterManager.

public void testNoLongerClusterManager() throws Exception {
    final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService(Settings.EMPTY, clusterService, () -> repositoriesService, () -> rerouteService);
    final Repository mockRepository = new FilterRepository(mock(Repository.class)) {

        @Override
        public IndexShardSnapshotStatus getShardSnapshotStatus(SnapshotId snapshotId, IndexId indexId, ShardId shardId) {
            return IndexShardSnapshotStatus.newDone(0L, 0L, 0, 0, 0L, randomNonNegativeLong(), null);
        }
    };
    when(repositoriesService.repository("_repo")).thenReturn(mockRepository);
    for (int i = 0; i < randomIntBetween(1, 10); i++) {
        final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
        final int nbShards = randomIntBetween(1, 5);
        applyClusterState("restore-indices-when-cluster-manager-" + indexName, clusterState -> addUnassignedShards(clusterState, indexName, nbShards));
    }
    applyClusterState("demote-current-cluster-manager", this::demoteClusterManagerNode);
    for (int i = 0; i < randomIntBetween(1, 10); i++) {
        final String indexName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
        final int nbShards = randomIntBetween(1, 5);
        applyClusterState("restore-indices-when-no-longer-cluster-manager-" + indexName, clusterState -> addUnassignedShards(clusterState, indexName, nbShards));
    }
    assertBusy(() -> {
        assertThat(snapshotsInfoService.numberOfKnownSnapshotShardSizes(), equalTo(0));
        assertThat(snapshotsInfoService.numberOfUnknownSnapshotShardSizes(), equalTo(0));
        assertThat(snapshotsInfoService.numberOfFailedSnapshotShardSizes(), equalTo(0));
    });
}