Examples with SnapshotInfo org.elasticsearch.snapshots.SnapshotInfo used on opensource projects

Example 11 with SnapshotInfo

use of org.elasticsearch.snapshots.SnapshotInfo in project elasticsearch by elastic.

the class BlobStoreRepository method deleteSnapshot.

@Override
public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId) {
    if (isReadOnly()) {
        throw new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository");
    }
    final RepositoryData repositoryData = getRepositoryData();
    List<String> indices = Collections.emptyList();
    SnapshotInfo snapshot = null;
    try {
        snapshot = getSnapshotInfo(snapshotId);
        indices = snapshot.indices();
    } catch (SnapshotMissingException ex) {
        throw ex;
    } catch (IllegalStateException | SnapshotException | ElasticsearchParseException ex) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("cannot read snapshot file [{}]", snapshotId), ex);
    }
    MetaData metaData = null;
    try {
        if (snapshot != null) {
            metaData = readSnapshotMetaData(snapshotId, snapshot.version(), repositoryData.resolveIndices(indices), true);
        } else {
            metaData = readSnapshotMetaData(snapshotId, null, repositoryData.resolveIndices(indices), true);
        }
    } catch (IOException | SnapshotException ex) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("cannot read metadata for snapshot [{}]", snapshotId), ex);
    }
    try {
        // Delete snapshot from the index file, since it is the maintainer of truth of active snapshots
        final RepositoryData updatedRepositoryData = repositoryData.removeSnapshot(snapshotId);
        writeIndexGen(updatedRepositoryData, repositoryStateId);
        // delete the snapshot file
        safeSnapshotBlobDelete(snapshot, snapshotId.getUUID());
        // delete the global metadata file
        safeGlobalMetaDataBlobDelete(snapshot, snapshotId.getUUID());
        // Now delete all indices
        for (String index : indices) {
            final IndexId indexId = repositoryData.resolveIndexId(index);
            BlobPath indexPath = basePath().add("indices").add(indexId.getId());
            BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
            try {
                indexMetaDataFormat.delete(indexMetaDataBlobContainer, snapshotId.getUUID());
            } catch (IOException ex) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete metadata for index [{}]", snapshotId, index), ex);
            }
            if (metaData != null) {
                IndexMetaData indexMetaData = metaData.index(index);
                if (indexMetaData != null) {
                    for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
                        try {
                            delete(snapshotId, snapshot.version(), indexId, new ShardId(indexMetaData.getIndex(), shardId));
                        } catch (SnapshotException ex) {
                            final int finalShardId = shardId;
                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete shard data for shard [{}][{}]", snapshotId, index, finalShardId), ex);
                        }
                    }
                }
            }
        }
        // cleanup indices that are no longer part of the repository
        final Collection<IndexId> indicesToCleanUp = Sets.newHashSet(repositoryData.getIndices().values());
        indicesToCleanUp.removeAll(updatedRepositoryData.getIndices().values());
        final BlobContainer indicesBlobContainer = blobStore().blobContainer(basePath().add("indices"));
        for (final IndexId indexId : indicesToCleanUp) {
            try {
                indicesBlobContainer.deleteBlob(indexId.getId());
            } catch (DirectoryNotEmptyException dnee) {
                // if the directory isn't empty for some reason, it will fail to clean up;
                // we'll ignore that and accept that cleanup didn't fully succeed.
                // since we are using UUIDs for path names, this won't be an issue for
                // snapshotting indices of the same name
                logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, but failed to clean up " + "its index folder due to the directory not being empty.", metadata.name(), indexId), dnee);
            } catch (IOException ioe) {
                // a different IOException occurred while trying to delete - will just log the issue for now
                logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, but failed to clean up " + "its index folder.", metadata.name(), indexId), ioe);
            }
        }
    } catch (IOException ex) {
        throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
    }
}

Example 12 with SnapshotInfo

use of org.elasticsearch.snapshots.SnapshotInfo in project elasticsearch by elastic.

the class BlobStoreRepository method finalizeSnapshot.

/**
     * {@inheritDoc}
     */
@Override
public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId, final List<IndexId> indices, final long startTime, final String failure, final int totalShards, final List<SnapshotShardFailure> shardFailures, final long repositoryStateId) {
    try {
        SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId, indices.stream().map(IndexId::getName).collect(Collectors.toList()), startTime, failure, System.currentTimeMillis(), totalShards, shardFailures);
        snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getUUID());
        final RepositoryData repositoryData = getRepositoryData();
        List<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
        if (!snapshotIds.contains(snapshotId)) {
            writeIndexGen(repositoryData.addSnapshot(snapshotId, indices), repositoryStateId);
        }
        return blobStoreSnapshot;
    } catch (IOException ex) {
        throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
    }
}

Example 13 with SnapshotInfo

use of org.elasticsearch.snapshots.SnapshotInfo in project elasticsearch by elastic.

the class RestSnapshotAction method buildTable.

private Table buildTable(RestRequest req, GetSnapshotsResponse getSnapshotsResponse) {
    Table table = getTableWithHeader(req);
    for (SnapshotInfo snapshotStatus : getSnapshotsResponse.getSnapshots()) {
        table.startRow();
        table.addCell(snapshotStatus.snapshotId().getName());
        table.addCell(snapshotStatus.state());
        table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.startTime(), TimeUnit.MILLISECONDS));
        table.addCell(dateFormat.print(snapshotStatus.startTime()));
        table.addCell(TimeUnit.SECONDS.convert(snapshotStatus.endTime(), TimeUnit.MILLISECONDS));
        table.addCell(dateFormat.print(snapshotStatus.endTime()));
        final long durationMillis;
        if (snapshotStatus.state() == SnapshotState.IN_PROGRESS) {
            durationMillis = System.currentTimeMillis() - snapshotStatus.startTime();
        } else {
            durationMillis = snapshotStatus.endTime() - snapshotStatus.startTime();
        }
        table.addCell(TimeValue.timeValueMillis(durationMillis));
        table.addCell(snapshotStatus.indices().size());
        table.addCell(snapshotStatus.successfulShards());
        table.addCell(snapshotStatus.failedShards());
        table.addCell(snapshotStatus.totalShards());
        table.addCell(snapshotStatus.reason());
        table.endRow();
    }
    return table;
}

Example 14 with SnapshotInfo

use of org.elasticsearch.snapshots.SnapshotInfo in project elasticsearch by elastic.

the class GetSnapshotsResponse method writeTo.

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeVInt(snapshots.size());
    for (SnapshotInfo snapshotInfo : snapshots) {
        snapshotInfo.writeTo(out);
    }
}

Example 15 with SnapshotInfo

use of org.elasticsearch.snapshots.SnapshotInfo in project elasticsearch by elastic.

the class RepositoryUpgradabilityIT method testRepositoryWorksWithCrossVersions.

/**
     * This tests that a repository can inter-operate with snapshots that both have and don't have a UUID,
     * namely when a repository was created in an older version with snapshots created in the old format
     * (only snapshot name, no UUID) and then the repository is loaded into newer versions where subsequent
     * snapshots have a name and a UUID.
     */
public void testRepositoryWorksWithCrossVersions() throws Exception {
    final List<String> repoVersions = listRepoVersions();
    // run the test for each supported version
    for (final String version : repoVersions) {
        final String repoName = "test-repo-" + version;
        logger.info("-->  creating repository [{}] for version [{}]", repoName, version);
        createRepository(version, repoName);
        logger.info("--> get the snapshots");
        final String originalIndex = "index-" + version;
        final Set<String> indices = Sets.newHashSet(originalIndex);
        final Set<SnapshotInfo> snapshotInfos = Sets.newHashSet(getSnapshots(repoName));
        assertThat(snapshotInfos.size(), equalTo(1));
        SnapshotInfo originalSnapshot = snapshotInfos.iterator().next();
        if (Version.fromString(version).before(Version.V_5_0_0_alpha1)) {
            assertThat(originalSnapshot.snapshotId(), equalTo(new SnapshotId("test_1", "test_1")));
        } else {
            assertThat(originalSnapshot.snapshotId().getName(), equalTo("test_1"));
            // it's a random UUID now
            assertNotNull(originalSnapshot.snapshotId().getUUID());
        }
        assertThat(Sets.newHashSet(originalSnapshot.indices()), equalTo(indices));
        logger.info("--> restore the original snapshot");
        final Set<String> restoredIndices = Sets.newHashSet(restoreSnapshot(repoName, originalSnapshot.snapshotId().getName()));
        assertThat(restoredIndices, equalTo(indices));
        // make sure it has documents
        for (final String searchIdx : restoredIndices) {
            assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().getTotalHits(), greaterThan(0L));
        }
        // delete so we can restore again later
        deleteIndices(restoredIndices);
        final String snapshotName2 = "test_2";
        logger.info("--> take a new snapshot of the old index");
        final int addedDocSize = 10;
        for (int i = 0; i < addedDocSize; i++) {
            index(originalIndex, "doc", Integer.toString(i), "foo", "new-bar-" + i);
        }
        refresh();
        snapshotInfos.add(createSnapshot(repoName, snapshotName2));
        logger.info("--> get the snapshots with the newly created snapshot [{}]", snapshotName2);
        Set<SnapshotInfo> snapshotInfosFromRepo = Sets.newHashSet(getSnapshots(repoName));
        assertThat(snapshotInfosFromRepo, equalTo(snapshotInfos));
        snapshotInfosFromRepo.forEach(snapshotInfo -> {
            assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(indices));
        });
        final String snapshotName3 = "test_3";
        final String indexName2 = "index2";
        logger.info("--> take a new snapshot with a new index");
        createIndex(indexName2);
        indices.add(indexName2);
        for (int i = 0; i < addedDocSize; i++) {
            index(indexName2, "doc", Integer.toString(i), "foo", "new-bar-" + i);
        }
        refresh();
        snapshotInfos.add(createSnapshot(repoName, snapshotName3));
        logger.info("--> get the snapshots with the newly created snapshot [{}]", snapshotName3);
        snapshotInfosFromRepo = Sets.newHashSet(getSnapshots(repoName));
        assertThat(snapshotInfosFromRepo, equalTo(snapshotInfos));
        snapshotInfosFromRepo.forEach(snapshotInfo -> {
            if (snapshotInfo.snapshotId().getName().equals(snapshotName3)) {
                assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(indices));
            } else {
                assertThat(Sets.newHashSet(snapshotInfo.indices()), equalTo(Sets.newHashSet(originalIndex)));
            }
        });
        // clean up indices
        deleteIndices(indices);
        logger.info("--> restore the old snapshot again");
        Set<String> oldRestoredIndices = Sets.newHashSet(restoreSnapshot(repoName, originalSnapshot.snapshotId().getName()));
        assertThat(oldRestoredIndices, equalTo(Sets.newHashSet(originalIndex)));
        for (final String searchIdx : oldRestoredIndices) {
            assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().getTotalHits(), greaterThanOrEqualTo((long) addedDocSize));
        }
        deleteIndices(oldRestoredIndices);
        logger.info("--> restore the new snapshot");
        Set<String> newSnapshotIndices = Sets.newHashSet(restoreSnapshot(repoName, snapshotName3));
        assertThat(newSnapshotIndices, equalTo(Sets.newHashSet(originalIndex, indexName2)));
        for (final String searchIdx : newSnapshotIndices) {
            assertThat(client().prepareSearch(searchIdx).setSize(0).get().getHits().getTotalHits(), greaterThanOrEqualTo((long) addedDocSize));
        }
        // clean up indices before starting again
        deleteIndices(newSnapshotIndices);
    }
}