Examples with IndexId org.elasticsearch.repositories.IndexId used on opensource projects

Example 6 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class BlobStoreRepository method deleteSnapshot.

@Override
public void deleteSnapshot(SnapshotId snapshotId, long repositoryStateId) {
    if (isReadOnly()) {
        throw new RepositoryException(metadata.name(), "cannot delete snapshot from a readonly repository");
    }
    final RepositoryData repositoryData = getRepositoryData();
    List<String> indices = Collections.emptyList();
    SnapshotInfo snapshot = null;
    try {
        snapshot = getSnapshotInfo(snapshotId);
        indices = snapshot.indices();
    } catch (SnapshotMissingException ex) {
        throw ex;
    } catch (IllegalStateException | SnapshotException | ElasticsearchParseException ex) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("cannot read snapshot file [{}]", snapshotId), ex);
    }
    MetaData metaData = null;
    try {
        if (snapshot != null) {
            metaData = readSnapshotMetaData(snapshotId, snapshot.version(), repositoryData.resolveIndices(indices), true);
        } else {
            metaData = readSnapshotMetaData(snapshotId, null, repositoryData.resolveIndices(indices), true);
        }
    } catch (IOException | SnapshotException ex) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("cannot read metadata for snapshot [{}]", snapshotId), ex);
    }
    try {
        // Delete snapshot from the index file, since it is the maintainer of truth of active snapshots
        final RepositoryData updatedRepositoryData = repositoryData.removeSnapshot(snapshotId);
        writeIndexGen(updatedRepositoryData, repositoryStateId);
        // delete the snapshot file
        safeSnapshotBlobDelete(snapshot, snapshotId.getUUID());
        // delete the global metadata file
        safeGlobalMetaDataBlobDelete(snapshot, snapshotId.getUUID());
        // Now delete all indices
        for (String index : indices) {
            final IndexId indexId = repositoryData.resolveIndexId(index);
            BlobPath indexPath = basePath().add("indices").add(indexId.getId());
            BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
            try {
                indexMetaDataFormat.delete(indexMetaDataBlobContainer, snapshotId.getUUID());
            } catch (IOException ex) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete metadata for index [{}]", snapshotId, index), ex);
            }
            if (metaData != null) {
                IndexMetaData indexMetaData = metaData.index(index);
                if (indexMetaData != null) {
                    for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
                        try {
                            delete(snapshotId, snapshot.version(), indexId, new ShardId(indexMetaData.getIndex(), shardId));
                        } catch (SnapshotException ex) {
                            final int finalShardId = shardId;
                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to delete shard data for shard [{}][{}]", snapshotId, index, finalShardId), ex);
                        }
                    }
                }
            }
        }
        // cleanup indices that are no longer part of the repository
        final Collection<IndexId> indicesToCleanUp = Sets.newHashSet(repositoryData.getIndices().values());
        indicesToCleanUp.removeAll(updatedRepositoryData.getIndices().values());
        final BlobContainer indicesBlobContainer = blobStore().blobContainer(basePath().add("indices"));
        for (final IndexId indexId : indicesToCleanUp) {
            try {
                indicesBlobContainer.deleteBlob(indexId.getId());
            } catch (DirectoryNotEmptyException dnee) {
                // if the directory isn't empty for some reason, it will fail to clean up;
                // we'll ignore that and accept that cleanup didn't fully succeed.
                // since we are using UUIDs for path names, this won't be an issue for
                // snapshotting indices of the same name
                logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, but failed to clean up " + "its index folder due to the directory not being empty.", metadata.name(), indexId), dnee);
            } catch (IOException ioe) {
                // a different IOException occurred while trying to delete - will just log the issue for now
                logger.debug((Supplier<?>) () -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, but failed to clean up " + "its index folder.", metadata.name(), indexId), ioe);
            }
        }
    } catch (IOException ex) {
        throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
    }
}

Example 7 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class BlobStoreRepository method finalizeSnapshot.

/**
     * {@inheritDoc}
     */
@Override
public SnapshotInfo finalizeSnapshot(final SnapshotId snapshotId, final List<IndexId> indices, final long startTime, final String failure, final int totalShards, final List<SnapshotShardFailure> shardFailures, final long repositoryStateId) {
    try {
        SnapshotInfo blobStoreSnapshot = new SnapshotInfo(snapshotId, indices.stream().map(IndexId::getName).collect(Collectors.toList()), startTime, failure, System.currentTimeMillis(), totalShards, shardFailures);
        snapshotFormat.write(blobStoreSnapshot, snapshotsBlobContainer, snapshotId.getUUID());
        final RepositoryData repositoryData = getRepositoryData();
        List<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
        if (!snapshotIds.contains(snapshotId)) {
            writeIndexGen(repositoryData.addSnapshot(snapshotId, indices), repositoryStateId);
        }
        return blobStoreSnapshot;
    } catch (IOException ex) {
        throw new RepositoryException(metadata.name(), "failed to update snapshot in repository", ex);
    }
}

Example 8 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class BlobStoreRepository method readSnapshotMetaData.

private MetaData readSnapshotMetaData(SnapshotId snapshotId, Version snapshotVersion, List<IndexId> indices, boolean ignoreIndexErrors) throws IOException {
    MetaData metaData;
    if (snapshotVersion == null) {
        // We can try detecting the version based on the metadata file format
        assert ignoreIndexErrors;
        if (globalMetaDataFormat.exists(snapshotsBlobContainer, snapshotId.getUUID())) {
            snapshotVersion = Version.CURRENT;
        } else {
            throw new SnapshotMissingException(metadata.name(), snapshotId);
        }
    }
    try {
        metaData = globalMetaDataFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
    } catch (NoSuchFileException ex) {
        throw new SnapshotMissingException(metadata.name(), snapshotId, ex);
    } catch (IOException ex) {
        throw new SnapshotException(metadata.name(), snapshotId, "failed to get snapshots", ex);
    }
    MetaData.Builder metaDataBuilder = MetaData.builder(metaData);
    for (IndexId index : indices) {
        BlobPath indexPath = basePath().add("indices").add(index.getId());
        BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
        try {
            metaDataBuilder.put(indexMetaDataFormat.read(indexMetaDataBlobContainer, snapshotId.getUUID()), false);
        } catch (ElasticsearchParseException | IOException ex) {
            if (ignoreIndexErrors) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, index.getName()), ex);
            } else {
                throw ex;
            }
        }
    }
    return metaDataBuilder.build();
}

Example 9 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class SnapshotShardsService method processIndexShardSnapshots.

/**
     * Checks if any new shards should be snapshotted on this node
     *
     * @param event cluster state changed event
     */
private void processIndexShardSnapshots(ClusterChangedEvent event) {
    SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
    Map<Snapshot, SnapshotShards> survivors = new HashMap<>();
    // First, remove snapshots that are no longer there
    for (Map.Entry<Snapshot, SnapshotShards> entry : shardSnapshots.entrySet()) {
        final Snapshot snapshot = entry.getKey();
        if (snapshotsInProgress != null && snapshotsInProgress.snapshot(snapshot) != null) {
            survivors.put(entry.getKey(), entry.getValue());
        } else {
            // state update, which is being processed here
            for (IndexShardSnapshotStatus snapshotStatus : entry.getValue().shards.values()) {
                if (snapshotStatus.stage() == Stage.INIT || snapshotStatus.stage() == Stage.STARTED) {
                    snapshotStatus.abort();
                }
            }
        }
    }
    // For now we will be mostly dealing with a single snapshot at a time but might have multiple simultaneously running
    // snapshots in the future
    Map<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> newSnapshots = new HashMap<>();
    // Now go through all snapshots and update existing or create missing
    final String localNodeId = event.state().nodes().getLocalNodeId();
    final DiscoveryNode masterNode = event.state().nodes().getMasterNode();
    final Map<Snapshot, Map<String, IndexId>> snapshotIndices = new HashMap<>();
    if (snapshotsInProgress != null) {
        for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
            snapshotIndices.put(entry.snapshot(), entry.indices().stream().collect(Collectors.toMap(IndexId::getName, Function.identity())));
            if (entry.state() == State.STARTED) {
                Map<ShardId, IndexShardSnapshotStatus> startedShards = new HashMap<>();
                SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                    // Add all new shards to start processing on
                    if (localNodeId.equals(shard.value.nodeId())) {
                        if (shard.value.state() == State.INIT && (snapshotShards == null || !snapshotShards.shards.containsKey(shard.key))) {
                            logger.trace("[{}] - Adding shard to the queue", shard.key);
                            startedShards.put(shard.key, new IndexShardSnapshotStatus());
                        }
                    }
                }
                if (!startedShards.isEmpty()) {
                    newSnapshots.put(entry.snapshot(), startedShards);
                    if (snapshotShards != null) {
                        // We already saw this snapshot but we need to add more started shards
                        Map<ShardId, IndexShardSnapshotStatus> shards = new HashMap<>();
                        // Put all shards that were already running on this node
                        shards.putAll(snapshotShards.shards);
                        // Put all newly started shards
                        shards.putAll(startedShards);
                        survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(shards)));
                    } else {
                        // Brand new snapshot that we haven't seen before
                        survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(startedShards)));
                    }
                }
            } else if (entry.state() == State.ABORTED) {
                // Abort all running shards for this snapshot
                SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                if (snapshotShards != null) {
                    for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                        IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.key);
                        if (snapshotStatus != null) {
                            switch(snapshotStatus.stage()) {
                                case INIT:
                                case STARTED:
                                    snapshotStatus.abort();
                                    break;
                                case FINALIZE:
                                    logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, letting it finish", entry.snapshot(), shard.key);
                                    break;
                                case DONE:
                                    logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, updating status on the master", entry.snapshot(), shard.key);
                                    updateIndexShardSnapshotStatus(entry.snapshot(), shard.key, new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                                    break;
                                case FAILURE:
                                    logger.debug("[{}] trying to cancel snapshot on the shard [{}] that has already failed, updating status on the master", entry.snapshot(), shard.key);
                                    updateIndexShardSnapshotStatus(entry.snapshot(), shard.key, new ShardSnapshotStatus(localNodeId, State.FAILED, snapshotStatus.failure()), masterNode);
                                    break;
                                default:
                                    throw new IllegalStateException("Unknown snapshot shard stage " + snapshotStatus.stage());
                            }
                        }
                    }
                }
            }
        }
    }
    // Update the list of snapshots that we saw and tried to started
    // If startup of these shards fails later, we don't want to try starting these shards again
    shutdownLock.lock();
    try {
        shardSnapshots = unmodifiableMap(survivors);
        if (shardSnapshots.isEmpty()) {
            // Notify all waiting threads that no more snapshots
            shutdownCondition.signalAll();
        }
    } finally {
        shutdownLock.unlock();
    }
    // We have new shards to starts
    if (newSnapshots.isEmpty() == false) {
        Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
        for (final Map.Entry<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> entry : newSnapshots.entrySet()) {
            Map<String, IndexId> indicesMap = snapshotIndices.get(entry.getKey());
            assert indicesMap != null;
            for (final Map.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : entry.getValue().entrySet()) {
                final ShardId shardId = shardEntry.getKey();
                try {
                    final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id());
                    final IndexId indexId = indicesMap.get(shardId.getIndexName());
                    assert indexId != null;
                    executor.execute(new AbstractRunnable() {

                        @Override
                        public void doRun() {
                            snapshot(indexShard, entry.getKey(), indexId, shardEntry.getValue());
                            updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                        }

                        @Override
                        public void onFailure(Exception e) {
                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to create snapshot", shardId, entry.getKey()), e);
                            updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                        }
                    });
                } catch (Exception e) {
                    updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                }
            }
        }
    }
}

Example 10 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class BlobStoreRepositoryTests method addRandomSnapshotsToRepoData.

private RepositoryData addRandomSnapshotsToRepoData(RepositoryData repoData, boolean inclIndices) {
    int numSnapshots = randomIntBetween(1, 20);
    for (int i = 0; i < numSnapshots; i++) {
        SnapshotId snapshotId = new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID());
        int numIndices = inclIndices ? randomIntBetween(0, 20) : 0;
        List<IndexId> indexIds = new ArrayList<>(numIndices);
        for (int j = 0; j < numIndices; j++) {
            indexIds.add(new IndexId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
        }
        repoData = repoData.addSnapshot(snapshotId, indexIds);
    }
    return repoData;
}