Examples with Snapshot org.elasticsearch.snapshots.Snapshot used on opensource projects

Example 6 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project elasticsearch by elastic.

the class IndexRecoveryIT method testSnapshotRecovery.

public void testSnapshotRecovery() throws Exception {
    logger.info("--> start node A");
    String nodeA = internalCluster().startNode();
    logger.info("--> create repository");
    assertAcked(client().admin().cluster().preparePutRepository(REPO_NAME).setType("fs").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", false)).get());
    ensureGreen();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot(REPO_NAME, SNAP_NAME).setWaitForCompletion(true).setIndices(INDEX_NAME).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(client().admin().cluster().prepareGetSnapshots(REPO_NAME).setSnapshots(SNAP_NAME).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet();
    logger.info("--> restore");
    RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot(REPO_NAME, SNAP_NAME).setWaitForCompletion(true).execute().actionGet();
    int totalShards = restoreSnapshotResponse.getRestoreInfo().totalShards();
    assertThat(totalShards, greaterThan(0));
    ensureGreen();
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates : response.shardRecoveryStates().entrySet()) {
        assertThat(indexRecoveryStates.getKey(), equalTo(INDEX_NAME));
        List<RecoveryState> recoveryStates = indexRecoveryStates.getValue();
        assertThat(recoveryStates.size(), equalTo(totalShards));
        for (RecoveryState recoveryState : recoveryStates) {
            SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(new Snapshot(REPO_NAME, createSnapshotResponse.getSnapshotInfo().snapshotId()), Version.CURRENT, INDEX_NAME);
            assertRecoveryState(recoveryState, 0, recoverySource, true, Stage.DONE, null, nodeA);
            validateIndexRecoveryState(recoveryState.getIndex());
        }
    }
}

Example 7 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project elasticsearch by elastic.

the class SnapshotStatus method readFrom.

@Override
public void readFrom(StreamInput in) throws IOException {
    snapshot = new Snapshot(in);
    state = State.fromValue(in.readByte());
    int size = in.readVInt();
    List<SnapshotIndexShardStatus> builder = new ArrayList<>();
    for (int i = 0; i < size; i++) {
        builder.add(SnapshotIndexShardStatus.readShardSnapshotStatus(in));
    }
    shards = Collections.unmodifiableList(builder);
    updateShardStats();
}

Example 8 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project elasticsearch by elastic.

the class TransportSnapshotsStatusAction method buildResponse.

private SnapshotsStatusResponse buildResponse(SnapshotsStatusRequest request, List<SnapshotsInProgress.Entry> currentSnapshotEntries, TransportNodesSnapshotsStatus.NodesSnapshotStatus nodeSnapshotStatuses) throws IOException {
    // First process snapshot that are currently processed
    List<SnapshotStatus> builder = new ArrayList<>();
    Set<String> currentSnapshotNames = new HashSet<>();
    if (!currentSnapshotEntries.isEmpty()) {
        Map<String, TransportNodesSnapshotsStatus.NodeSnapshotStatus> nodeSnapshotStatusMap;
        if (nodeSnapshotStatuses != null) {
            nodeSnapshotStatusMap = nodeSnapshotStatuses.getNodesMap();
        } else {
            nodeSnapshotStatusMap = new HashMap<>();
        }
        for (SnapshotsInProgress.Entry entry : currentSnapshotEntries) {
            currentSnapshotNames.add(entry.snapshot().getSnapshotId().getName());
            List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
            for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shardEntry : entry.shards()) {
                SnapshotsInProgress.ShardSnapshotStatus status = shardEntry.value;
                if (status.nodeId() != null) {
                    // We should have information about this shard from the shard:
                    TransportNodesSnapshotsStatus.NodeSnapshotStatus nodeStatus = nodeSnapshotStatusMap.get(status.nodeId());
                    if (nodeStatus != null) {
                        Map<ShardId, SnapshotIndexShardStatus> shardStatues = nodeStatus.status().get(entry.snapshot());
                        if (shardStatues != null) {
                            SnapshotIndexShardStatus shardStatus = shardStatues.get(shardEntry.key);
                            if (shardStatus != null) {
                                // We have full information about this shard
                                shardStatusBuilder.add(shardStatus);
                                continue;
                            }
                        }
                    }
                }
                final SnapshotIndexShardStage stage;
                switch(shardEntry.value.state()) {
                    case FAILED:
                    case ABORTED:
                    case MISSING:
                        stage = SnapshotIndexShardStage.FAILURE;
                        break;
                    case INIT:
                    case WAITING:
                    case STARTED:
                        stage = SnapshotIndexShardStage.STARTED;
                        break;
                    case SUCCESS:
                        stage = SnapshotIndexShardStage.DONE;
                        break;
                    default:
                        throw new IllegalArgumentException("Unknown snapshot state " + shardEntry.value.state());
                }
                SnapshotIndexShardStatus shardStatus = new SnapshotIndexShardStatus(shardEntry.key, stage);
                shardStatusBuilder.add(shardStatus);
            }
            builder.add(new SnapshotStatus(entry.snapshot(), entry.state(), Collections.unmodifiableList(shardStatusBuilder)));
        }
    }
    // Now add snapshots on disk that are not currently running
    final String repositoryName = request.repository();
    if (Strings.hasText(repositoryName) && request.snapshots() != null && request.snapshots().length > 0) {
        final Set<String> requestedSnapshotNames = Sets.newHashSet(request.snapshots());
        final RepositoryData repositoryData = snapshotsService.getRepositoryData(repositoryName);
        final Map<String, SnapshotId> matchedSnapshotIds = repositoryData.getAllSnapshotIds().stream().filter(s -> requestedSnapshotNames.contains(s.getName())).collect(Collectors.toMap(SnapshotId::getName, Function.identity()));
        for (final String snapshotName : request.snapshots()) {
            if (currentSnapshotNames.contains(snapshotName)) {
                // we've already found this snapshot in the current snapshot entries, so skip over
                continue;
            }
            SnapshotId snapshotId = matchedSnapshotIds.get(snapshotName);
            if (snapshotId == null) {
                // neither in the current snapshot entries nor found in the repository
                if (request.ignoreUnavailable()) {
                    // ignoring unavailable snapshots, so skip over
                    logger.debug("snapshot status request ignoring snapshot [{}], not found in repository [{}]", snapshotName, repositoryName);
                    continue;
                } else {
                    throw new SnapshotMissingException(repositoryName, snapshotName);
                }
            } else if (repositoryData.getIncompatibleSnapshotIds().contains(snapshotId)) {
                throw new SnapshotException(repositoryName, snapshotName, "cannot get the status for an incompatible snapshot");
            }
            SnapshotInfo snapshotInfo = snapshotsService.snapshot(repositoryName, snapshotId);
            List<SnapshotIndexShardStatus> shardStatusBuilder = new ArrayList<>();
            if (snapshotInfo.state().completed()) {
                Map<ShardId, IndexShardSnapshotStatus> shardStatues = snapshotsService.snapshotShards(request.repository(), snapshotInfo);
                for (Map.Entry<ShardId, IndexShardSnapshotStatus> shardStatus : shardStatues.entrySet()) {
                    shardStatusBuilder.add(new SnapshotIndexShardStatus(shardStatus.getKey(), shardStatus.getValue()));
                }
                final SnapshotsInProgress.State state;
                switch(snapshotInfo.state()) {
                    case FAILED:
                        state = SnapshotsInProgress.State.FAILED;
                        break;
                    case SUCCESS:
                    case PARTIAL:
                        // Translating both PARTIAL and SUCCESS to SUCCESS for now
                        // TODO: add the differentiation on the metadata level in the next major release
                        state = SnapshotsInProgress.State.SUCCESS;
                        break;
                    default:
                        throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state());
                }
                builder.add(new SnapshotStatus(new Snapshot(repositoryName, snapshotId), state, Collections.unmodifiableList(shardStatusBuilder)));
            }
        }
    }
    return new SnapshotsStatusResponse(Collections.unmodifiableList(builder));
}

Example 9 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project elasticsearch by elastic.

the class ThrottlingAllocationTests method createRecoveryStateAndInitalizeAllocations.

private ClusterState createRecoveryStateAndInitalizeAllocations(MetaData metaData, TestGatewayAllocator gatewayAllocator) {
    DiscoveryNode node1 = newNode("node1");
    MetaData.Builder metaDataBuilder = new MetaData.Builder(metaData);
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    for (ObjectCursor<IndexMetaData> cursor : metaData.indices().values()) {
        Index index = cursor.value.getIndex();
        IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(cursor.value);
        final int recoveryType = randomInt(5);
        if (recoveryType <= 4) {
            addInSyncAllocationIds(index, indexMetaDataBuilder, gatewayAllocator, node1);
        }
        IndexMetaData indexMetaData = indexMetaDataBuilder.build();
        metaDataBuilder.put(indexMetaData, false);
        switch(recoveryType) {
            case 0:
                routingTableBuilder.addAsRecovery(indexMetaData);
                break;
            case 1:
                routingTableBuilder.addAsFromCloseToOpen(indexMetaData);
                break;
            case 2:
                routingTableBuilder.addAsFromDangling(indexMetaData);
                break;
            case 3:
                routingTableBuilder.addAsNewRestore(indexMetaData, new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT, indexMetaData.getIndex().getName()), new IntHashSet());
                break;
            case 4:
                routingTableBuilder.addAsRestore(indexMetaData, new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT, indexMetaData.getIndex().getName()));
                break;
            case 5:
                routingTableBuilder.addAsNew(indexMetaData);
                break;
            default:
                throw new IndexOutOfBoundsException();
        }
    }
    return ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(DiscoveryNodes.builder().add(node1)).metaData(metaDataBuilder.build()).routingTable(routingTableBuilder.build()).build();
}

Example 10 with Snapshot

use of org.elasticsearch.snapshots.Snapshot in project elasticsearch by elastic.

the class FilterAllocationDeciderTests method createInitialClusterState.

private ClusterState createInitialClusterState(AllocationService service, Settings settings) {
    RecoverySource.Type recoveryType = randomFrom(RecoverySource.Type.EMPTY_STORE, RecoverySource.Type.LOCAL_SHARDS, RecoverySource.Type.SNAPSHOT);
    MetaData.Builder metaData = MetaData.builder();
    final Settings.Builder indexSettings = settings(Version.CURRENT).put(settings);
    final IndexMetaData sourceIndex;
    if (recoveryType == RecoverySource.Type.LOCAL_SHARDS) {
        //put a fake closed source index
        sourceIndex = IndexMetaData.builder("sourceIndex").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(0).putInSyncAllocationIds(0, Collections.singleton("aid0")).putInSyncAllocationIds(1, Collections.singleton("aid1")).build();
        metaData.put(sourceIndex, false);
        indexSettings.put(INDEX_SHRINK_SOURCE_UUID.getKey(), sourceIndex.getIndexUUID());
        indexSettings.put(INDEX_SHRINK_SOURCE_NAME.getKey(), sourceIndex.getIndex().getName());
    } else {
        sourceIndex = null;
    }
    final IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder("idx").settings(indexSettings).numberOfShards(1).numberOfReplicas(1);
    if (recoveryType == RecoverySource.Type.SNAPSHOT) {
        indexMetaDataBuilder.putInSyncAllocationIds(0, Collections.singleton("_snapshot_restore"));
    }
    final IndexMetaData indexMetaData = indexMetaDataBuilder.build();
    metaData.put(indexMetaData, false);
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    switch(recoveryType) {
        case EMPTY_STORE:
            routingTableBuilder.addAsNew(indexMetaData);
            break;
        case SNAPSHOT:
            routingTableBuilder.addAsRestore(indexMetaData, new RecoverySource.SnapshotRecoverySource(new Snapshot("repository", new SnapshotId("snapshot_name", "snapshot_uuid")), Version.CURRENT, indexMetaData.getIndex().getName()));
            break;
        case LOCAL_SHARDS:
            routingTableBuilder.addAsFromCloseToOpen(sourceIndex);
            routingTableBuilder.addAsNew(indexMetaData);
            break;
        default:
            throw new UnsupportedOperationException(recoveryType + " is not supported");
    }
    RoutingTable routingTable = routingTableBuilder.build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    return service.reroute(clusterState, "reroute", false);
}