Examples with Snapshot org.opensearch.snapshots.Snapshot used on opensource projects

Example 16 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class MetadataIndexStateServiceTests method addSnapshotIndex.

private static ClusterState addSnapshotIndex(final String index, final int numShards, final int numReplicas, final ClusterState state) {
    ClusterState newState = addOpenedIndex(index, numShards, numReplicas, state);
    final ImmutableOpenMap.Builder<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shardsBuilder = ImmutableOpenMap.builder();
    for (ShardRouting shardRouting : newState.routingTable().index(index).randomAllActiveShardsIt()) {
        shardsBuilder.put(shardRouting.shardId(), new SnapshotsInProgress.ShardSnapshotStatus(shardRouting.currentNodeId(), "1"));
    }
    final Snapshot snapshot = new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5)));
    final SnapshotsInProgress.Entry entry = new SnapshotsInProgress.Entry(snapshot, randomBoolean(), false, SnapshotsInProgress.State.INIT, Collections.singletonList(new IndexId(index, index)), Collections.emptyList(), randomNonNegativeLong(), randomLong(), shardsBuilder.build(), null, SnapshotInfoTests.randomUserMetadata(), VersionUtils.randomVersion(random()));
    return ClusterState.builder(newState).putCustom(SnapshotsInProgress.TYPE, SnapshotsInProgress.of(Collections.singletonList(entry))).build();
}

Example 17 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class PrimaryShardAllocatorTests method getRestoreRoutingAllocation.

private RoutingAllocation getRestoreRoutingAllocation(AllocationDeciders allocationDeciders, Long shardSize, String... allocIds) {
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder(shardId.getIndexName()).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).putInSyncAllocationIds(0, Sets.newHashSet(allocIds))).build();
    final Snapshot snapshot = new Snapshot("test", new SnapshotId("test", UUIDs.randomBase64UUID()));
    RoutingTable routingTable = RoutingTable.builder().addAsRestore(metadata.index(shardId.getIndex()), new SnapshotRecoverySource(UUIDs.randomBase64UUID(), snapshot, Version.CURRENT, new IndexId(shardId.getIndexName(), UUIDs.randomBase64UUID(random())))).build();
    ClusterState state = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(routingTable).nodes(DiscoveryNodes.builder().add(node1).add(node2).add(node3)).build();
    return new RoutingAllocation(allocationDeciders, new RoutingNodes(state, false), state, null, new SnapshotShardSizeInfo(ImmutableOpenMap.of()) {

        @Override
        public Long getShardSize(ShardRouting shardRouting) {
            return shardSize;
        }
    }, System.nanoTime());
}

Example 18 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class ThrottlingAllocationTests method createRecoveryStateAndInitializeAllocations.

private ClusterState createRecoveryStateAndInitializeAllocations(final Metadata metadata, final TestGatewayAllocator gatewayAllocator, final TestSnapshotsInfoService snapshotsInfoService, final Integer inputRecoveryType) {
    DiscoveryNode node1 = newNode("node1");
    Metadata.Builder metadataBuilder = new Metadata.Builder(metadata);
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    Snapshot snapshot = new Snapshot("repo", new SnapshotId("snap", "randomId"));
    Set<String> snapshotIndices = new HashSet<>();
    String restoreUUID = UUIDs.randomBase64UUID();
    for (ObjectCursor<IndexMetadata> cursor : metadata.indices().values()) {
        Index index = cursor.value.getIndex();
        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(cursor.value);
        final int recoveryType = inputRecoveryType == null ? randomInt(5) : inputRecoveryType.intValue();
        if (recoveryType <= 4) {
            addInSyncAllocationIds(index, indexMetadataBuilder, gatewayAllocator, node1);
        }
        IndexMetadata indexMetadata = indexMetadataBuilder.build();
        metadataBuilder.put(indexMetadata, false);
        switch(recoveryType) {
            case 0:
                routingTableBuilder.addAsRecovery(indexMetadata);
                break;
            case 1:
                routingTableBuilder.addAsFromCloseToOpen(indexMetadata);
                break;
            case 2:
                routingTableBuilder.addAsFromDangling(indexMetadata);
                break;
            case 3:
                snapshotIndices.add(index.getName());
                routingTableBuilder.addAsNewRestore(indexMetadata, new SnapshotRecoverySource(restoreUUID, snapshot, Version.CURRENT, new IndexId(indexMetadata.getIndex().getName(), UUIDs.randomBase64UUID(random()))), new IntHashSet());
                break;
            case 4:
                snapshotIndices.add(index.getName());
                routingTableBuilder.addAsRestore(indexMetadata, new SnapshotRecoverySource(restoreUUID, snapshot, Version.CURRENT, new IndexId(indexMetadata.getIndex().getName(), UUIDs.randomBase64UUID(random()))));
                break;
            case 5:
                routingTableBuilder.addAsNew(indexMetadata);
                break;
            default:
                throw new IndexOutOfBoundsException();
        }
    }
    final RoutingTable routingTable = routingTableBuilder.build();
    final ImmutableOpenMap.Builder<String, ClusterState.Custom> restores = ImmutableOpenMap.builder();
    if (snapshotIndices.isEmpty() == false) {
        // Some indices are restored from snapshot, the RestoreInProgress must be set accordingly
        ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> restoreShards = ImmutableOpenMap.builder();
        for (ShardRouting shard : routingTable.allShards()) {
            if (shard.primary() && shard.recoverySource().getType() == RecoverySource.Type.SNAPSHOT) {
                final ShardId shardId = shard.shardId();
                restoreShards.put(shardId, new RestoreInProgress.ShardRestoreStatus(node1.getId(), RestoreInProgress.State.INIT));
                // Also set the snapshot shard size
                final SnapshotRecoverySource recoverySource = (SnapshotRecoverySource) shard.recoverySource();
                final long shardSize = randomNonNegativeLong();
                snapshotsInfoService.addSnapshotShardSize(recoverySource.snapshot(), recoverySource.index(), shardId, shardSize);
            }
        }
        RestoreInProgress.Entry restore = new RestoreInProgress.Entry(restoreUUID, snapshot, RestoreInProgress.State.INIT, new ArrayList<>(snapshotIndices), restoreShards.build());
        restores.put(RestoreInProgress.TYPE, new RestoreInProgress.Builder().add(restore).build());
    }
    return ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(DiscoveryNodes.builder().add(node1)).metadata(metadataBuilder.build()).routingTable(routingTable).customs(restores.build()).build();
}

Example 19 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class NodeVersionAllocationDeciderTests method testRestoreDoesNotAllocateSnapshotOnOlderNodes.

public void testRestoreDoesNotAllocateSnapshotOnOlderNodes() {
    final DiscoveryNode newNode = new DiscoveryNode("newNode", buildNewFakeTransportAddress(), emptyMap(), CLUSTER_MANAGER_DATA_ROLES, Version.CURRENT);
    final DiscoveryNode oldNode1 = new DiscoveryNode("oldNode1", buildNewFakeTransportAddress(), emptyMap(), CLUSTER_MANAGER_DATA_ROLES, VersionUtils.getPreviousVersion());
    final DiscoveryNode oldNode2 = new DiscoveryNode("oldNode2", buildNewFakeTransportAddress(), emptyMap(), CLUSTER_MANAGER_DATA_ROLES, VersionUtils.getPreviousVersion());
    final Snapshot snapshot = new Snapshot("rep1", new SnapshotId("snp1", UUIDs.randomBase64UUID()));
    final IndexId indexId = new IndexId("test", UUIDs.randomBase64UUID(random()));
    final int numberOfShards = randomIntBetween(1, 3);
    final IndexMetadata.Builder indexMetadata = IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(numberOfShards).numberOfReplicas(randomIntBetween(0, 3));
    for (int i = 0; i < numberOfShards; i++) {
        indexMetadata.putInSyncAllocationIds(i, Collections.singleton("_test_"));
    }
    Metadata metadata = Metadata.builder().put(indexMetadata).build();
    final ImmutableOpenMap.Builder<InternalSnapshotsInfoService.SnapshotShard, Long> snapshotShardSizes = ImmutableOpenMap.builder(numberOfShards);
    final Index index = metadata.index("test").getIndex();
    for (int i = 0; i < numberOfShards; i++) {
        final ShardId shardId = new ShardId(index, i);
        snapshotShardSizes.put(new InternalSnapshotsInfoService.SnapshotShard(snapshot, indexId, shardId), randomNonNegativeLong());
    }
    ClusterState state = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(RoutingTable.builder().addAsRestore(metadata.index("test"), new SnapshotRecoverySource(UUIDs.randomBase64UUID(), snapshot, Version.CURRENT, indexId)).build()).nodes(DiscoveryNodes.builder().add(newNode).add(oldNode1).add(oldNode2)).build();
    AllocationDeciders allocationDeciders = new AllocationDeciders(Arrays.asList(new ReplicaAfterPrimaryActiveAllocationDecider(), new NodeVersionAllocationDecider()));
    AllocationService strategy = new MockAllocationService(allocationDeciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE, () -> new SnapshotShardSizeInfo(snapshotShardSizes.build()));
    state = strategy.reroute(state, new AllocationCommands(), true, false).getClusterState();
    // Make sure that primary shards are only allocated on the new node
    for (int i = 0; i < numberOfShards; i++) {
        assertEquals("newNode", state.routingTable().index("test").getShards().get(i).primaryShard().currentNodeId());
    }
}

Example 20 with Snapshot

use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.

the class SnapshotStatusTests method testToString.

public void testToString() throws Exception {
    SnapshotsInProgress.State state = randomFrom(SnapshotsInProgress.State.values());
    String uuid = UUIDs.randomBase64UUID();
    SnapshotId id = new SnapshotId("test-snap", uuid);
    Snapshot snapshot = new Snapshot("test-repo", id);
    String indexName = randomAlphaOfLengthBetween(3, 50);
    int shardId = randomInt();
    ShardId testShardId = ShardId.fromString("[" + indexName + "][" + shardId + "]");
    SnapshotIndexShardStage shardStage = randomFrom(SnapshotIndexShardStage.values());
    SnapshotIndexShardStatus snapshotIndexShardStatus = new SnapshotIndexShardStatus(testShardId, shardStage);
    List<SnapshotIndexShardStatus> snapshotIndexShardStatuses = new ArrayList<>();
    snapshotIndexShardStatuses.add(snapshotIndexShardStatus);
    boolean includeGlobalState = randomBoolean();
    SnapshotStatus status = new SnapshotStatus(snapshot, state, snapshotIndexShardStatuses, includeGlobalState, 0L, 0L);
    int initializingShards = 0;
    int startedShards = 0;
    int finalizingShards = 0;
    int doneShards = 0;
    int failedShards = 0;
    int totalShards = 1;
    switch(shardStage) {
        case INIT:
            initializingShards++;
            break;
        case STARTED:
            startedShards++;
            break;
        case FINALIZE:
            finalizingShards++;
            break;
        case DONE:
            doneShards++;
            break;
        case FAILURE:
            failedShards++;
            break;
        default:
            break;
    }
    String expected = "{\n" + "  \"snapshot\" : \"test-snap\",\n" + "  \"repository\" : \"test-repo\",\n" + "  \"uuid\" : \"" + uuid + "\",\n" + "  \"state\" : \"" + state.toString() + "\",\n" + "  \"include_global_state\" : " + includeGlobalState + ",\n" + "  \"shards_stats\" : {\n" + "    \"initializing\" : " + initializingShards + ",\n" + "    \"started\" : " + startedShards + ",\n" + "    \"finalizing\" : " + finalizingShards + ",\n" + "    \"done\" : " + doneShards + ",\n" + "    \"failed\" : " + failedShards + ",\n" + "    \"total\" : " + totalShards + "\n" + "  },\n" + "  \"stats\" : {\n" + "    \"incremental\" : {\n" + "      \"file_count\" : 0,\n" + "      \"size_in_bytes\" : 0\n" + "    },\n" + "    \"total\" : {\n" + "      \"file_count\" : 0,\n" + "      \"size_in_bytes\" : 0\n" + "    },\n" + "    \"start_time_in_millis\" : 0,\n" + "    \"time_in_millis\" : 0\n" + "  },\n" + "  \"indices\" : {\n" + "    \"" + indexName + "\" : {\n" + "      \"shards_stats\" : {\n" + "        \"initializing\" : " + initializingShards + ",\n" + "        \"started\" : " + startedShards + ",\n" + "        \"finalizing\" : " + finalizingShards + ",\n" + "        \"done\" : " + doneShards + ",\n" + "        \"failed\" : " + failedShards + ",\n" + "        \"total\" : " + totalShards + "\n" + "      },\n" + "      \"stats\" : {\n" + "        \"incremental\" : {\n" + "          \"file_count\" : 0,\n" + "          \"size_in_bytes\" : 0\n" + "        },\n" + "        \"total\" : {\n" + "          \"file_count\" : 0,\n" + "          \"size_in_bytes\" : 0\n" + "        },\n" + "        \"start_time_in_millis\" : 0,\n" + "        \"time_in_millis\" : 0\n" + "      },\n" + "      \"shards\" : {\n" + "        \"" + shardId + "\" : {\n" + "          \"stage\" : \"" + shardStage.toString() + "\",\n" + "          \"stats\" : {\n" + "            \"incremental\" : {\n" + "              \"file_count\" : 0,\n" + "              \"size_in_bytes\" : 0\n" + "            },\n" + "            \"total\" : {\n" + "              \"file_count\" : 0,\n" + "              \"size_in_bytes\" : 0\n" + "            },\n" + "            \"start_time_in_millis\" : 0,\n" + "            \"time_in_millis\" : 0\n" + "          }\n" + "        }\n" + "      }\n" + "    }\n" + "  }\n" + "}";
    assertEquals(expected, status.toString());
}