Examples with GroupedActionListener org.elasticsearch.action.support.GroupedActionListener used on opensource projects

Example 6 with GroupedActionListener

use of org.elasticsearch.action.support.GroupedActionListener in project crate by crate.

the class ReplicationTracker method createMissingPeerRecoveryRetentionLeases.

/**
 * Create any required peer-recovery retention leases that do not currently exist because we just did a rolling upgrade from a version
 * prior to {@link Version#V_4_3_0} that does not create peer-recovery retention leases.
 */
public synchronized void createMissingPeerRecoveryRetentionLeases(ActionListener<Void> listener) {
    if (hasAllPeerRecoveryRetentionLeases == false) {
        final List<ShardRouting> shardRoutings = routingTable.assignedShards();
        final GroupedActionListener<ReplicationResponse> groupedActionListener = new GroupedActionListener<>(ActionListener.wrap(vs -> {
            setHasAllPeerRecoveryRetentionLeases();
            listener.onResponse(null);
        }, listener::onFailure), shardRoutings.size());
        for (ShardRouting shardRouting : shardRoutings) {
            if (retentionLeases.contains(getPeerRecoveryRetentionLeaseId(shardRouting))) {
                groupedActionListener.onResponse(null);
            } else {
                final CheckpointState checkpointState = checkpoints.get(shardRouting.allocationId().getId());
                if (checkpointState.tracked == false) {
                    groupedActionListener.onResponse(null);
                } else {
                    logger.trace("createMissingPeerRecoveryRetentionLeases: adding missing lease for {}", shardRouting);
                    try {
                        addPeerRecoveryRetentionLease(shardRouting.currentNodeId(), Math.max(SequenceNumbers.NO_OPS_PERFORMED, checkpointState.globalCheckpoint), groupedActionListener);
                    } catch (Exception e) {
                        groupedActionListener.onFailure(e);
                    }
                }
            }
        }
    } else {
        logger.trace("createMissingPeerRecoveryRetentionLeases: nothing to do");
        listener.onResponse(null);
    }
}

Example 7 with GroupedActionListener

use of org.elasticsearch.action.support.GroupedActionListener in project crate by crate.

the class BlobStoreRepository method cleanupStaleBlobs.

/**
 * Cleans up stale blobs directly under the repository root as well as all indices paths that aren't referenced by any existing
 * snapshots. This method is only to be called directly after a new {@link RepositoryData} was written to the repository and with
 * parameters {@code foundIndices}, {@code rootBlobs}
 *
 * @param foundIndices all indices blob containers found in the repository before {@code newRepoData} was written
 * @param rootBlobs    all blobs found directly under the repository root
 * @param newRepoData  new repository data that was just written
 * @param listener     listener to invoke with the combined long of all blobs removed in this operation
 */
private void cleanupStaleBlobs(Map<String, BlobContainer> foundIndices, Map<String, BlobMetadata> rootBlobs, RepositoryData newRepoData, ActionListener<Long> listener) {
    final GroupedActionListener<Long> groupedListener = new GroupedActionListener<>(ActionListener.wrap(deleteResults -> {
        long deletes = 0;
        for (Long result : deleteResults) {
            deletes += result;
        }
        listener.onResponse(deletes);
    }, listener::onFailure), 2);
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    executor.execute(ActionRunnable.supply(groupedListener, () -> {
        List<String> deletedBlobs = cleanupStaleRootFiles(staleRootBlobs(newRepoData, rootBlobs.keySet()));
        return (long) deletedBlobs.size();
    }));
    final Set<String> survivingIndexIds = newRepoData.getIndices().values().stream().map(IndexId::getId).collect(Collectors.toSet());
    executor.execute(ActionRunnable.supply(groupedListener, () -> cleanupStaleIndices(foundIndices, survivingIndexIds)));
}

Example 8 with GroupedActionListener

use of org.elasticsearch.action.support.GroupedActionListener in project crate by crate.

the class BlobStoreRepository method writeUpdatedShardMetadataAndComputeDeletes.

// updates the shard state metadata for shards of a snapshot that is to be deleted. Also computes the files to be cleaned up.
private void writeUpdatedShardMetadataAndComputeDeletes(SnapshotId snapshotId, RepositoryData oldRepositoryData, boolean useUUIDs, ActionListener<Collection<ShardSnapshotMetaDeleteResult>> onAllShardsCompleted) {
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    final List<IndexId> indices = oldRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotId);
    if (indices.isEmpty()) {
        onAllShardsCompleted.onResponse(Collections.emptyList());
        return;
    }
    // Listener that flattens out the delete results for each index
    final ActionListener<Collection<ShardSnapshotMetaDeleteResult>> deleteIndexMetadataListener = new GroupedActionListener<>(ActionListener.map(onAllShardsCompleted, res -> res.stream().flatMap(Collection::stream).collect(Collectors.toList())), indices.size());
    for (IndexId indexId : indices) {
        final Set<SnapshotId> survivingSnapshots = oldRepositoryData.getSnapshots(indexId).stream().filter(id -> id.equals(snapshotId) == false).collect(Collectors.toSet());
        executor.execute(ActionRunnable.wrap(deleteIndexMetadataListener, deleteIdxMetaListener -> {
            final StepListener<IndexMetadata> snapshotIndexMetadataListener = new StepListener<>();
            try {
                getSnapshotIndexMetadata(snapshotId, indexId, snapshotIndexMetadataListener);
            } catch (Exception ex) {
                LOGGER.warn(() -> new ParameterizedMessage("[{}] [{}] failed to read metadata for index", snapshotId, indexId.getName()), ex);
                // Just invoke the listener without any shard generations to count it down, this index will be cleaned up
                // by the stale data cleanup in the end.
                // TODO: Getting here means repository corruption. We should find a way of dealing with this instead of just ignoring
                // it and letting the cleanup deal with it.
                deleteIdxMetaListener.onResponse(null);
                return;
            }
            snapshotIndexMetadataListener.whenComplete(indexMetadata -> {
                final int shardCount = indexMetadata.getNumberOfShards();
                assert shardCount > 0 : "index did not have positive shard count, get [" + shardCount + "]";
                // Listener for collecting the results of removing the snapshot from each shard's metadata in the current index
                final ActionListener<ShardSnapshotMetaDeleteResult> allShardsListener = new GroupedActionListener<>(deleteIdxMetaListener, shardCount);
                final Index index = indexMetadata.getIndex();
                for (int shardId = 0; shardId < indexMetadata.getNumberOfShards(); shardId++) {
                    final ShardId shard = new ShardId(index, shardId);
                    executor.execute(new AbstractRunnable() {

                        @Override
                        protected void doRun() throws Exception {
                            final BlobContainer shardContainer = shardContainer(indexId, shard);
                            final Set<String> blobs = getShardBlobs(shard, shardContainer);
                            final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots;
                            final String newGen;
                            if (useUUIDs) {
                                newGen = UUIDs.randomBase64UUID();
                                blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, oldRepositoryData.shardGenerations().getShardGen(indexId, shard.getId())).v1();
                            } else {
                                Tuple<BlobStoreIndexShardSnapshots, Long> tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer);
                                newGen = Long.toString(tuple.v2() + 1);
                                blobStoreIndexShardSnapshots = tuple.v1();
                            }
                            allShardsListener.onResponse(deleteFromShardSnapshotMeta(survivingSnapshots, indexId, shard, snapshotId, shardContainer, blobs, blobStoreIndexShardSnapshots, newGen));
                        }

                        @Override
                        public void onFailure(Exception ex) {
                            LOGGER.warn(() -> new ParameterizedMessage("[{}] failed to delete shard data for shard [{}][{}]", snapshotId, indexId.getName(), shard.id()), ex);
                            // Just passing null here to count down the listener instead of failing it, the stale data left behind
                            // here will be retried in the next delete or repository cleanup
                            allShardsListener.onResponse(null);
                        }
                    });
                }
            }, onAllShardsCompleted::onFailure);
        }));
    }
}

Example 9 with GroupedActionListener

use of org.elasticsearch.action.support.GroupedActionListener in project crate by crate.

the class SnapshotsService method hasOldVersionSnapshots.

public void hasOldVersionSnapshots(String repositoryName, RepositoryData repositoryData, ActionListener<Boolean> listener, @Nullable SnapshotId excluded) {
    final Collection<SnapshotId> snapshotIds = repositoryData.getSnapshotIds();
    var snapshotIdsFiltered = snapshotIds.stream().filter(snapshotId -> snapshotId.equals(excluded) == false).toList();
    if (snapshotIdsFiltered.isEmpty()) {
        listener.onResponse(false);
    } else {
        if (repositoryData.shardGenerations().totalShards() > 0) {
            listener.onResponse(false);
        } else {
            final Repository repository = repositoriesService.repository(repositoryName);
            GroupedActionListener<Boolean> allSnapshotIdsListener = new GroupedActionListener<>(ActionListener.wrap(oldFormatResults -> {
                boolean hasOldFormatSnapshots = false;
                for (boolean res : oldFormatResults) {
                    hasOldFormatSnapshots = res;
                }
                if (hasOldFormatSnapshots == false || repositoryData.shardGenerations().totalShards() == 0) {
                    listener.onResponse(hasOldFormatSnapshots);
                } else {
                    listener.onFailure(new RuntimeException("Found non-empty shard generations [" + repositoryData.shardGenerations() + "] but repository contained old version snapshots"));
                }
            }, (e) -> {
                if (e instanceof SnapshotMissingException) {
                    LOGGER.warn("Failed to load snapshot metadata, assuming repository is in old format", e);
                    listener.onResponse(true);
                } else {
                    listener.onFailure(e);
                }
            }), snapshotIdsFiltered.size());
            for (var snapshotId : snapshotIdsFiltered) {
                final Version known = repositoryData.getVersion(snapshotId);
                if (known == null) {
                    repository.getSnapshotInfo(snapshotId, ActionListener.delegateFailure(allSnapshotIdsListener, (delegate, snapshotInfo) -> {
                        var version = snapshotInfo.version();
                        if (version != null) {
                            delegate.onResponse(version.before(SHARD_GEN_IN_REPO_DATA_VERSION));
                        } else {
                            delegate.onResponse(false);
                        }
                    }));
                } else {
                    allSnapshotIdsListener.onResponse(known.before(SHARD_GEN_IN_REPO_DATA_VERSION));
                }
            }
        }
    }
}

Example 10 with GroupedActionListener

use of org.elasticsearch.action.support.GroupedActionListener in project crate by crate.

the class SnapshotsService method snapshots.

/**
 * Returns a list of snapshots from repository sorted by snapshot creation date
 *
 * @param repositoryName repository name
 * @param snapshotIds       snapshots for which to fetch snapshot information
 * @param ignoreUnavailable if true, snapshots that could not be read will only be logged with a warning,
 *                          if false, they will throw an error
 * @return list of snapshots
 */
public void snapshots(final String repositoryName, final List<SnapshotId> snapshotIds, final boolean ignoreUnavailable, ActionListener<Collection<SnapshotInfo>> listener) {
    final Set<SnapshotInfo> snapshotSet = new HashSet<>();
    final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds);
    // first, look at the snapshots in progress
    final List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, snapshotIdsToIterate.stream().map(SnapshotId::getName).collect(Collectors.toList()));
    for (SnapshotsInProgress.Entry entry : entries) {
        snapshotSet.add(inProgressSnapshot(entry));
        snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId());
    }
    if (snapshotIdsToIterate.isEmpty()) {
        listener.onResponse(snapshotSet);
        return;
    }
    // then, look in the repository
    final Repository repository = repositoriesService.repository(repositoryName);
    GroupedActionListener<SnapshotInfo> snapshotInfosListener = new GroupedActionListener<>(ActionListener.wrap(snapshotInfos -> {
        final ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotSet);
        snapshotList.addAll(snapshotInfos);
        CollectionUtil.timSort(snapshotList);
        listener.onResponse(unmodifiableList(snapshotList));
    }, listener::onFailure), snapshotIdsToIterate.size());
    for (SnapshotId snapshotId : snapshotIdsToIterate) {
        StepListener<SnapshotInfo> future = new StepListener<>();
        future.whenComplete(snapshotInfosListener::onResponse, (ex) -> {
            if (ignoreUnavailable) {
                LOGGER.warn(() -> new ParameterizedMessage("failed to get snapshot [{}]", snapshotId), ex);
                snapshotInfosListener.onResponse(null);
            } else {
                snapshotInfosListener.onFailure(ex);
            }
        });
        repository.getSnapshotInfo(snapshotId, future);
    }
}