Examples with IndexId org.elasticsearch.repositories.IndexId used on opensource projects

Example 21 with IndexId

use of org.elasticsearch.repositories.IndexId in project crate by crate.

the class BlobStoreRepositoryRestoreTests method testSnapshotWithConflictingName.

public void testSnapshotWithConflictingName() throws IOException {
    final IndexId indexId = new IndexId(randomAlphaOfLength(10), UUIDs.randomBase64UUID());
    final ShardId shardId = new ShardId(indexId.getName(), indexId.getId(), 0);
    IndexShard shard = newShard(shardId, true);
    try {
        // index documents in the shards
        final int numDocs = scaledRandomIntBetween(1, 500);
        recoverShardFromStore(shard);
        for (int i = 0; i < numDocs; i++) {
            indexDoc(shard, Integer.toString(i));
            if (rarely()) {
                flushShard(shard, false);
            }
        }
        assertDocCount(shard, numDocs);
        // snapshot the shard
        final Repository repository = createRepository();
        final Snapshot snapshot = new Snapshot(repository.getMetadata().name(), new SnapshotId(randomAlphaOfLength(10), "_uuid"));
        final String shardGen = snapshotShard(shard, snapshot, repository);
        assertNotNull(shardGen);
        final Snapshot snapshotWithSameName = new Snapshot(repository.getMetadata().name(), new SnapshotId(snapshot.getSnapshotId().getName(), "_uuid2"));
        final PlainActionFuture<SnapshotInfo> future = PlainActionFuture.newFuture();
        repository.finalizeSnapshot(snapshot.getSnapshotId(), ShardGenerations.builder().put(indexId, 0, shardGen).build(), 0L, null, 1, Collections.emptyList(), -1L, false, Metadata.builder().put(shard.indexSettings().getIndexMetadata(), false).build(), true, future);
        future.actionGet();
        IndexShardSnapshotFailedException isfe = expectThrows(IndexShardSnapshotFailedException.class, () -> snapshotShard(shard, snapshotWithSameName, repository));
        assertThat(isfe.getMessage(), containsString("Duplicate snapshot name"));
    } finally {
        if (shard != null && shard.state() != IndexShardState.CLOSED) {
            try {
                shard.close("test", false);
            } finally {
                IOUtils.close(shard.store());
            }
        }
    }
}

Example 22 with IndexId

use of org.elasticsearch.repositories.IndexId in project crate by crate.

the class BlobStoreRepositoryTest method addRandomSnapshotsToRepoData.

private static RepositoryData addRandomSnapshotsToRepoData(RepositoryData repoData, boolean inclIndices) {
    int numSnapshots = randomIntBetween(1, 20);
    for (int i = 0; i < numSnapshots; i++) {
        SnapshotId snapshotId = new SnapshotId(randomAlphaOfLength(8), UUIDs.randomBase64UUID());
        int numIndices = inclIndices ? randomIntBetween(0, 20) : 0;
        final ShardGenerations.Builder builder = ShardGenerations.builder();
        for (int j = 0; j < numIndices; j++) {
            builder.put(new IndexId(randomAlphaOfLength(8), UUIDs.randomBase64UUID()), 0, "1");
        }
        repoData = repoData.addSnapshot(snapshotId, randomFrom(SnapshotState.SUCCESS, SnapshotState.PARTIAL, SnapshotState.FAILED), Version.CURRENT, builder.build());
    }
    return repoData;
}

Example 23 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class BlobStoreRepository method deleteSnapshot.

@Override
public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId) {
    if (isReadOnly()) {
        throw new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository");
    }
    final RepositoryData repositoryData = getRepositoryData();
    List<String> indices = Collections.emptyList();
    SnapshotInfo snapshot = null;
    try {
        snapshot = getSnapshotInfo(snapshotId);
        indices = snapshot.indices();
    } catch (SnapshotMissingException ex) {
        throw ex;
    } catch (IllegalStateException | SnapshotException | ElasticsearchParseException ex) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("cannot read snapshot file [{}]", snapshotId), ex);
    }
    MetaData metaData = null;
    try {
        if (snapshot != null) {
            metaData = readSnapshotMetaData(snapshotId, snapshot.version(), repositoryData.resolveIndices(indices), true);
        } else {
            metaData = readSnapshotMetaData(snapshotId, null, repositoryData.resolveIndices(indices), true);
        }
    } catch (IOException | SnapshotException ex) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("cannot read metadata for snapshot [{}]", snapshotId), ex);
    }
    try {
        // Delete snapshot from the index file, since it is the maintainer of truth of active snapshots
        final RepositoryData updatedRepositoryData = repositoryData.removeSnapshot(snapshotId);
        writeIndexGen(updatedRepositoryData, repositoryStateId);
        // delete the snapshot file
        safeSnapshotBlobDelete(snapshot, snapshotId.getUUID());
        // delete the global metadata file
        safeGlobalMetaDataBlobDelete(snapshot, snapshotId.getUUID());
        // Now delete all indices
        for (String index : indices) {
            final IndexId indexId = repositoryData.resolveIndexId(index);
            BlobPath indexPath = basePath().add("indices").add(indexId.getId());
            BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
            try {
                indexMetaDataFormat.delete(indexMetaDataBlobContainer, snapshotId.getUUID());
            } catch (IOException ex) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete metadata for index [{}]", snapshotId, index), ex);
            }
            if (metaData != null) {
                IndexMetaData indexMetaData = metaData.index(index);
                if (indexMetaData != null) {
                    for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
                        try {
                            delete(snapshotId, snapshot.version(), indexId, new ShardId(indexMetaData.getIndex(), shardId));
                        } catch (SnapshotException ex) {
                            final int finalShardId = shardId;
                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete shard data for shard [{}][{}]", snapshotId, index, finalShardId), ex);
                        }
                    }
                }
            }
        }
        // cleanup indices that are no longer part of the repository
        final Collection<IndexId> indicesToCleanUp = Sets.newHashSet(repositoryData.getIndices().values());
        indicesToCleanUp.removeAll(updatedRepositoryData.getIndices().values());
        final BlobContainer indicesBlobContainer = blobStore().blobContainer(basePath().add("indices"));
        for (final IndexId indexId : indicesToCleanUp) {
            try {
                indicesBlobContainer.deleteBlob(indexId.getId());
            } catch (DirectoryNotEmptyException dnee) {
                // if the directory isn't empty for some reason, it will fail to clean up;
                // we'll ignore that and accept that cleanup didn't fully succeed.
                // since we are using UUIDs for path names, this won't be an issue for
                // snapshotting indices of the same name
                logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, but failed to clean up " + "its index folder due to the directory not being empty.", metadata.name(), indexId), dnee);
            } catch (IOException ioe) {
                // a different IOException occurred while trying to delete - will just log the issue for now
                logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, but failed to clean up " + "its index folder.", metadata.name(), indexId), ioe);
            }
        }
    } catch (IOException ex) {
        throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
    }
}

Example 24 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class BlobStoreRepository method finalizeSnapshot.

/**
     * {@inheritDoc}
     */
@Override
public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId, final List<IndexId> indices, final long startTime, final String failure, final int totalShards, final List<SnapshotShardFailure> shardFailures, final long repositoryStateId) {
    try {
        SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId, indices.stream().map(IndexId::getName).collect(Collectors.toList()), startTime, failure, System.currentTimeMillis(), totalShards, shardFailures);
        snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getUUID());
        final RepositoryData repositoryData = getRepositoryData();
        List<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
        if (!snapshotIds.contains(snapshotId)) {
            writeIndexGen(repositoryData.addSnapshot(snapshotId, indices), repositoryStateId);
        }
        return blobStoreSnapshot;
    } catch (IOException ex) {
        throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
    }
}

Example 25 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class BlobStoreRepository method readSnapshotMetaData.

private MetaData readSnapshotMetaData(SnapshotId snapshotId, Version snapshotVersion, List<IndexId> indices, boolean ignoreIndexErrors) throws IOException {
    MetaData metaData;
    if (snapshotVersion == null) {
        // We can try detecting the version based on the metadata file format
        assert ignoreIndexErrors;
        if (globalMetaDataFormat.exists(snapshotsBlobContainer, snapshotId.getUUID())) {
            snapshotVersion = Version.CURRENT;
        } else {
            throw new SnapshotMissingException(metadata.name(), snapshotId);
        }
    }
    try {
        metaData = globalMetaDataFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
    } catch (NoSuchFileException ex) {
        throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
    } catch (IOException ex) {
        throw new SnapshotException(metadata.name(), snapshotId, "failed to get snapshots", ex);
    }
    MetaData.Builder metaDataBuilder = MetaData.builder(metaData);
    for (IndexId index : indices) {
        BlobPath indexPath = basePath().add("indices").add(index.getId());
        BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
        try {
            metaDataBuilder.put(indexMetaDataFormat.read(indexMetaDataBlobContainer, snapshotId.getUUID()), false);
        } catch (ElasticsearchParseException | IOException ex) {
            if (ignoreIndexErrors) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, index.getName()), ex);
            } else {
                throw ex;
            }
        }
    }
    return metaDataBuilder.build();
}