Examples with GroupedActionListener org.opensearch.action.support.GroupedActionListener used on opensource projects

Example 1 with GroupedActionListener

use of org.opensearch.action.support.GroupedActionListener in project OpenSearch by opensearch-project.

the class AzureBlobContainer method deleteBlobsIgnoringIfNotExists.

@Override
public void deleteBlobsIgnoringIfNotExists(List<String> blobNames) throws IOException {
    final PlainActionFuture<Void> result = PlainActionFuture.newFuture();
    if (blobNames.isEmpty()) {
        result.onResponse(null);
    } else {
        final GroupedActionListener<Void> listener = new GroupedActionListener<>(ActionListener.map(result, v -> null), blobNames.size());
        final ExecutorService executor = threadPool.executor(AzureRepositoryPlugin.REPOSITORY_THREAD_POOL_NAME);
        // TODO: Upgrade to newer non-blocking Azure SDK 11 and execute delete requests in parallel that way.
        for (String blobName : blobNames) {
            executor.execute(ActionRunnable.run(listener, () -> {
                logger.trace("deleteBlob({})", blobName);
                try {
                    blobStore.deleteBlob(buildKey(blobName));
                } catch (BlobStorageException e) {
                    if (e.getStatusCode() != HttpURLConnection.HTTP_NOT_FOUND) {
                        throw new IOException(e);
                    }
                } catch (URISyntaxException e) {
                    throw new IOException(e);
                }
            }));
        }
    }
    try {
        result.actionGet();
    } catch (Exception e) {
        throw new IOException("Exception during bulk delete", e);
    }
}

Example 2 with GroupedActionListener

use of org.opensearch.action.support.GroupedActionListener in project OpenSearch by opensearch-project.

the class DiskThresholdMonitor method onNewInfo.

public void onNewInfo(ClusterInfo info) {
    // all ClusterInfo updates are processed and never ignored
    if (checkInProgress.compareAndSet(false, true) == false) {
        logger.info("skipping monitor as a check is already in progress");
        return;
    }
    final ImmutableOpenMap<String, DiskUsage> usages = info.getNodeLeastAvailableDiskUsages();
    if (usages == null) {
        logger.trace("skipping monitor as no disk usage information is available");
        checkFinished();
        return;
    }
    logger.trace("processing new cluster info");
    boolean reroute = false;
    String explanation = "";
    final long currentTimeMillis = currentTimeMillisSupplier.getAsLong();
    // Clean up nodes that have been removed from the cluster
    final ObjectLookupContainer<String> nodes = usages.keys();
    cleanUpRemovedNodes(nodes, nodesOverLowThreshold);
    cleanUpRemovedNodes(nodes, nodesOverHighThreshold);
    cleanUpRemovedNodes(nodes, nodesOverHighThresholdAndRelocating);
    final ClusterState state = clusterStateSupplier.get();
    final Set<String> indicesToMarkReadOnly = new HashSet<>();
    RoutingNodes routingNodes = state.getRoutingNodes();
    Set<String> indicesNotToAutoRelease = new HashSet<>();
    markNodesMissingUsageIneligibleForRelease(routingNodes, usages, indicesNotToAutoRelease);
    final List<DiskUsage> usagesOverHighThreshold = new ArrayList<>();
    for (final ObjectObjectCursor<String, DiskUsage> entry : usages) {
        final String node = entry.key;
        final DiskUsage usage = entry.value;
        final RoutingNode routingNode = routingNodes.node(node);
        if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) {
            nodesOverLowThreshold.add(node);
            nodesOverHighThreshold.add(node);
            nodesOverHighThresholdAndRelocating.remove(node);
            if (routingNode != null) {
                // might be temporarily null if the ClusterInfoService and the ClusterService are out of step
                for (ShardRouting routing : routingNode) {
                    String indexName = routing.index().getName();
                    indicesToMarkReadOnly.add(indexName);
                    indicesNotToAutoRelease.add(indexName);
                }
            }
            logger.warn("flood stage disk watermark [{}] exceeded on {}, all indices on this node will be marked read-only", diskThresholdSettings.describeFloodStageThreshold(), usage);
            continue;
        }
        if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
            if (routingNode != null) {
                // might be temporarily null if the ClusterInfoService and the ClusterService are out of step
                for (ShardRouting routing : routingNode) {
                    String indexName = routing.index().getName();
                    indicesNotToAutoRelease.add(indexName);
                }
            }
        }
        final long reservedSpace = info.getReservedSpace(usage.getNodeId(), usage.getPath()).getTotal();
        final DiskUsage usageWithReservedSpace = new DiskUsage(usage.getNodeId(), usage.getNodeName(), usage.getPath(), usage.getTotalBytes(), Math.max(0L, usage.getFreeBytes() - reservedSpace));
        if (usageWithReservedSpace.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() || usageWithReservedSpace.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
            nodesOverLowThreshold.add(node);
            nodesOverHighThreshold.add(node);
            if (lastRunTimeMillis.get() <= currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) {
                reroute = true;
                explanation = "high disk watermark exceeded on one or more nodes";
                usagesOverHighThreshold.add(usage);
            // will log about this node when the reroute completes
            } else {
                logger.debug("high disk watermark exceeded on {} but an automatic reroute has occurred " + "in the last [{}], skipping reroute", node, diskThresholdSettings.getRerouteInterval());
            }
        } else if (usageWithReservedSpace.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdLow().getBytes() || usageWithReservedSpace.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdLow()) {
            nodesOverHighThresholdAndRelocating.remove(node);
            final boolean wasUnderLowThreshold = nodesOverLowThreshold.add(node);
            final boolean wasOverHighThreshold = nodesOverHighThreshold.remove(node);
            assert (wasUnderLowThreshold && wasOverHighThreshold) == false;
            if (wasUnderLowThreshold) {
                logger.info("low disk watermark [{}] exceeded on {}, replicas will not be assigned to this node", diskThresholdSettings.describeLowThreshold(), usage);
            } else if (wasOverHighThreshold) {
                logger.info("high disk watermark [{}] no longer exceeded on {}, but low disk watermark [{}] is still exceeded", diskThresholdSettings.describeHighThreshold(), usage, diskThresholdSettings.describeLowThreshold());
            }
        } else {
            nodesOverHighThresholdAndRelocating.remove(node);
            if (nodesOverLowThreshold.contains(node)) {
                // if we reroute now.
                if (lastRunTimeMillis.get() <= currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) {
                    reroute = true;
                    explanation = "one or more nodes has gone under the high or low watermark";
                    nodesOverLowThreshold.remove(node);
                    nodesOverHighThreshold.remove(node);
                    logger.info("low disk watermark [{}] no longer exceeded on {}", diskThresholdSettings.describeLowThreshold(), usage);
                } else {
                    logger.debug("{} has gone below a disk threshold, but an automatic reroute has occurred " + "in the last [{}], skipping reroute", node, diskThresholdSettings.getRerouteInterval());
                }
            }
        }
    }
    final ActionListener<Void> listener = new GroupedActionListener<>(ActionListener.wrap(this::checkFinished), 3);
    if (reroute) {
        logger.debug("rerouting shards: [{}]", explanation);
        rerouteService.reroute("disk threshold monitor", Priority.HIGH, ActionListener.wrap(reroutedClusterState -> {
            for (DiskUsage diskUsage : usagesOverHighThreshold) {
                final RoutingNode routingNode = reroutedClusterState.getRoutingNodes().node(diskUsage.getNodeId());
                final DiskUsage usageIncludingRelocations;
                final long relocatingShardsSize;
                if (routingNode != null) {
                    // might be temporarily null if the ClusterInfoService and the ClusterService are out of step
                    relocatingShardsSize = sizeOfRelocatingShards(routingNode, diskUsage, info, reroutedClusterState);
                    usageIncludingRelocations = new DiskUsage(diskUsage.getNodeId(), diskUsage.getNodeName(), diskUsage.getPath(), diskUsage.getTotalBytes(), diskUsage.getFreeBytes() - relocatingShardsSize);
                } else {
                    usageIncludingRelocations = diskUsage;
                    relocatingShardsSize = 0L;
                }
                if (usageIncludingRelocations.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() || usageIncludingRelocations.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
                    nodesOverHighThresholdAndRelocating.remove(diskUsage.getNodeId());
                    logger.warn("high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; " + "currently relocating away shards totalling [{}] bytes; the node is expected to continue to exceed " + "the high disk watermark when these relocations are complete", diskThresholdSettings.describeHighThreshold(), diskUsage, -relocatingShardsSize);
                } else if (nodesOverHighThresholdAndRelocating.add(diskUsage.getNodeId())) {
                    logger.info("high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; " + "currently relocating away shards totalling [{}] bytes; the node is expected to be below the high " + "disk watermark when these relocations are complete", diskThresholdSettings.describeHighThreshold(), diskUsage, -relocatingShardsSize);
                } else {
                    logger.debug("high disk watermark [{}] exceeded on {}, shards will be relocated away from this node; " + "currently relocating away shards totalling [{}] bytes", diskThresholdSettings.describeHighThreshold(), diskUsage, -relocatingShardsSize);
                }
            }
            setLastRunTimeMillis();
            listener.onResponse(null);
        }, e -> {
            logger.debug("reroute failed", e);
            setLastRunTimeMillis();
            listener.onFailure(e);
        }));
    } else {
        logger.trace("no reroute required");
        listener.onResponse(null);
    }
    final Set<String> indicesToAutoRelease = StreamSupport.stream(state.routingTable().indicesRouting().spliterator(), false).map(c -> c.key).filter(index -> indicesNotToAutoRelease.contains(index) == false).filter(index -> state.getBlocks().hasIndexBlock(index, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK)).collect(Collectors.toSet());
    if (indicesToAutoRelease.isEmpty() == false) {
        if (diskThresholdSettings.isAutoReleaseIndexEnabled()) {
            logger.info("releasing read-only-allow-delete block on indices: [{}]", indicesToAutoRelease);
            updateIndicesReadOnly(indicesToAutoRelease, listener, false);
        } else {
            deprecationLogger.deprecate(DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY.replace(".", "_"), "[{}] will be removed in version {}", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, LegacyESVersion.V_7_4_0.major + 1);
            logger.debug("[{}] disabled, not releasing read-only-allow-delete block on indices: [{}]", DiskThresholdSettings.AUTO_RELEASE_INDEX_ENABLED_KEY, indicesToAutoRelease);
            listener.onResponse(null);
        }
    } else {
        logger.trace("no auto-release required");
        listener.onResponse(null);
    }
    indicesToMarkReadOnly.removeIf(index -> state.getBlocks().indexBlocked(ClusterBlockLevel.WRITE, index));
    logger.trace("marking indices as read-only: [{}]", indicesToMarkReadOnly);
    if (indicesToMarkReadOnly.isEmpty() == false) {
        updateIndicesReadOnly(indicesToMarkReadOnly, listener, true);
    } else {
        listener.onResponse(null);
    }
}

Example 3 with GroupedActionListener

use of org.opensearch.action.support.GroupedActionListener in project OpenSearch by opensearch-project.

the class SnapshotsService method startCloning.

/**
 * Determine the number of shards in each index of a clone operation and update the cluster state accordingly.
 *
 * @param repository     repository to run operation on
 * @param cloneEntry     clone operation in the cluster state
 */
private void startCloning(Repository repository, SnapshotsInProgress.Entry cloneEntry) {
    final List<IndexId> indices = cloneEntry.indices();
    final SnapshotId sourceSnapshot = cloneEntry.source();
    final Snapshot targetSnapshot = cloneEntry.snapshot();
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    // Exception handler for IO exceptions with loading index and repo metadata
    final Consumer<Exception> onFailure = e -> {
        initializingClones.remove(targetSnapshot);
        logger.info(() -> new ParameterizedMessage("Failed to start snapshot clone [{}]", cloneEntry), e);
        removeFailedSnapshotFromClusterState(targetSnapshot, e, null, null);
    };
    // 1. step, load SnapshotInfo to make sure that source snapshot was successful for the indices we want to clone
    // TODO: we could skip this step for snapshots with state SUCCESS
    final StepListener<SnapshotInfo> snapshotInfoListener = new StepListener<>();
    executor.execute(ActionRunnable.supply(snapshotInfoListener, () -> repository.getSnapshotInfo(sourceSnapshot)));
    final StepListener<Collection<Tuple<IndexId, Integer>>> allShardCountsListener = new StepListener<>();
    final GroupedActionListener<Tuple<IndexId, Integer>> shardCountListener = new GroupedActionListener<>(allShardCountsListener, indices.size());
    snapshotInfoListener.whenComplete(snapshotInfo -> {
        for (IndexId indexId : indices) {
            if (RestoreService.failed(snapshotInfo, indexId.getName())) {
                throw new SnapshotException(targetSnapshot, "Can't clone index [" + indexId + "] because its snapshot was not successful.");
            }
        }
        // 2. step, load the number of shards we have in each index to be cloned from the index metadata.
        repository.getRepositoryData(ActionListener.wrap(repositoryData -> {
            for (IndexId index : indices) {
                executor.execute(ActionRunnable.supply(shardCountListener, () -> {
                    final IndexMetadata metadata = repository.getSnapshotIndexMetaData(repositoryData, sourceSnapshot, index);
                    return Tuple.tuple(index, metadata.getNumberOfShards());
                }));
            }
        }, onFailure));
    }, onFailure);
    // 3. step, we have all the shard counts, now update the cluster state to have clone jobs in the snap entry
    allShardCountsListener.whenComplete(counts -> repository.executeConsistentStateUpdate(repoData -> new ClusterStateUpdateTask() {

        private SnapshotsInProgress.Entry updatedEntry;

        @Override
        public ClusterState execute(ClusterState currentState) {
            final SnapshotsInProgress snapshotsInProgress = currentState.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY);
            final List<SnapshotsInProgress.Entry> updatedEntries = new ArrayList<>(snapshotsInProgress.entries());
            boolean changed = false;
            final String localNodeId = currentState.nodes().getLocalNodeId();
            final String repoName = cloneEntry.repository();
            final ShardGenerations shardGenerations = repoData.shardGenerations();
            for (int i = 0; i < updatedEntries.size(); i++) {
                if (cloneEntry.snapshot().equals(updatedEntries.get(i).snapshot())) {
                    final ImmutableOpenMap.Builder<RepositoryShardId, ShardSnapshotStatus> clonesBuilder = ImmutableOpenMap.builder();
                    final InFlightShardSnapshotStates inFlightShardStates = InFlightShardSnapshotStates.forRepo(repoName, snapshotsInProgress.entries());
                    for (Tuple<IndexId, Integer> count : counts) {
                        for (int shardId = 0; shardId < count.v2(); shardId++) {
                            final RepositoryShardId repoShardId = new RepositoryShardId(count.v1(), shardId);
                            final String indexName = repoShardId.indexName();
                            if (inFlightShardStates.isActive(indexName, shardId)) {
                                clonesBuilder.put(repoShardId, ShardSnapshotStatus.UNASSIGNED_QUEUED);
                            } else {
                                clonesBuilder.put(repoShardId, new ShardSnapshotStatus(localNodeId, inFlightShardStates.generationForShard(repoShardId.index(), shardId, shardGenerations)));
                            }
                        }
                    }
                    updatedEntry = cloneEntry.withClones(clonesBuilder.build());
                    updatedEntries.set(i, updatedEntry);
                    changed = true;
                    break;
                }
            }
            return updateWithSnapshots(currentState, changed ? SnapshotsInProgress.of(updatedEntries) : null, null);
        }

        @Override
        public void onFailure(String source, Exception e) {
            initializingClones.remove(targetSnapshot);
            logger.info(() -> new ParameterizedMessage("Failed to start snapshot clone [{}]", cloneEntry), e);
            failAllListenersOnMasterFailOver(e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            initializingClones.remove(targetSnapshot);
            if (updatedEntry != null) {
                final Snapshot target = updatedEntry.snapshot();
                final SnapshotId sourceSnapshot = updatedEntry.source();
                for (ObjectObjectCursor<RepositoryShardId, ShardSnapshotStatus> indexClone : updatedEntry.clones()) {
                    final ShardSnapshotStatus shardStatusBefore = indexClone.value;
                    if (shardStatusBefore.state() != ShardState.INIT) {
                        continue;
                    }
                    final RepositoryShardId repoShardId = indexClone.key;
                    runReadyClone(target, sourceSnapshot, shardStatusBefore, repoShardId, repository);
                }
            } else {
                // Extremely unlikely corner case of master failing over between between starting the clone and
                // starting shard clones.
                logger.warn("Did not find expected entry [{}] in the cluster state", cloneEntry);
            }
        }
    }, "start snapshot clone", onFailure), onFailure);
}

Example 4 with GroupedActionListener

use of org.opensearch.action.support.GroupedActionListener in project OpenSearch by opensearch-project.

the class TaskCancellationService method setBanOnNodes.

private void setBanOnNodes(String reason, boolean waitForCompletion, CancellableTask task, Collection<DiscoveryNode> childNodes, ActionListener<Void> listener) {
    if (childNodes.isEmpty()) {
        listener.onResponse(null);
        return;
    }
    final TaskId taskId = new TaskId(localNodeId(), task.getId());
    logger.trace("cancelling child tasks of [{}] on child nodes {}", taskId, childNodes);
    GroupedActionListener<Void> groupedListener = new GroupedActionListener<>(ActionListener.map(listener, r -> null), childNodes.size());
    final BanParentTaskRequest banRequest = BanParentTaskRequest.createSetBanParentTaskRequest(taskId, reason, waitForCompletion);
    for (DiscoveryNode node : childNodes) {
        transportService.sendRequest(node, BAN_PARENT_ACTION_NAME, banRequest, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {

            @Override
            public void handleResponse(TransportResponse.Empty response) {
                logger.trace("sent ban for tasks with the parent [{}] to the node [{}]", taskId, node);
                groupedListener.onResponse(null);
            }

            @Override
            public void handleException(TransportException exp) {
                assert ExceptionsHelper.unwrapCause(exp) instanceof OpenSearchSecurityException == false;
                logger.warn("Cannot send ban for tasks with the parent [{}] to the node [{}]", taskId, node);
                groupedListener.onFailure(exp);
            }
        });
    }
}

Example 5 with GroupedActionListener

use of org.opensearch.action.support.GroupedActionListener in project OpenSearch by opensearch-project.

the class BlobStoreRepository method writeUpdatedShardMetaDataAndComputeDeletes.

// updates the shard state metadata for shards of a snapshot that is to be deleted. Also computes the files to be cleaned up.
private void writeUpdatedShardMetaDataAndComputeDeletes(Collection<SnapshotId> snapshotIds, RepositoryData oldRepositoryData, boolean useUUIDs, ActionListener<Collection<ShardSnapshotMetaDeleteResult>> onAllShardsCompleted) {
    final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
    final List<IndexId> indices = oldRepositoryData.indicesToUpdateAfterRemovingSnapshot(snapshotIds);
    if (indices.isEmpty()) {
        onAllShardsCompleted.onResponse(Collections.emptyList());
        return;
    }
    // Listener that flattens out the delete results for each index
    final ActionListener<Collection<ShardSnapshotMetaDeleteResult>> deleteIndexMetadataListener = new GroupedActionListener<>(ActionListener.map(onAllShardsCompleted, res -> res.stream().flatMap(Collection::stream).collect(Collectors.toList())), indices.size());
    for (IndexId indexId : indices) {
        final Set<SnapshotId> survivingSnapshots = oldRepositoryData.getSnapshots(indexId).stream().filter(id -> snapshotIds.contains(id) == false).collect(Collectors.toSet());
        final StepListener<Collection<Integer>> shardCountListener = new StepListener<>();
        final Collection<String> indexMetaGenerations = snapshotIds.stream().map(id -> oldRepositoryData.indexMetaDataGenerations().indexMetaBlobId(id, indexId)).collect(Collectors.toSet());
        final ActionListener<Integer> allShardCountsListener = new GroupedActionListener<>(shardCountListener, indexMetaGenerations.size());
        final BlobContainer indexContainer = indexContainer(indexId);
        for (String indexMetaGeneration : indexMetaGenerations) {
            executor.execute(ActionRunnable.supply(allShardCountsListener, () -> {
                try {
                    return INDEX_METADATA_FORMAT.read(indexContainer, indexMetaGeneration, namedXContentRegistry).getNumberOfShards();
                } catch (Exception ex) {
                    logger.warn(() -> new ParameterizedMessage("[{}] [{}] failed to read metadata for index", indexMetaGeneration, indexId.getName()), ex);
                    // ignoring it and letting the cleanup deal with it.
                    return null;
                }
            }));
        }
        shardCountListener.whenComplete(counts -> {
            final int shardCount = counts.stream().mapToInt(i -> i).max().orElse(0);
            if (shardCount == 0) {
                deleteIndexMetadataListener.onResponse(null);
                return;
            }
            // Listener for collecting the results of removing the snapshot from each shard's metadata in the current index
            final ActionListener<ShardSnapshotMetaDeleteResult> allShardsListener = new GroupedActionListener<>(deleteIndexMetadataListener, shardCount);
            for (int shardId = 0; shardId < shardCount; shardId++) {
                final int finalShardId = shardId;
                executor.execute(new AbstractRunnable() {

                    @Override
                    protected void doRun() throws Exception {
                        final BlobContainer shardContainer = shardContainer(indexId, finalShardId);
                        final Set<String> blobs = shardContainer.listBlobs().keySet();
                        final BlobStoreIndexShardSnapshots blobStoreIndexShardSnapshots;
                        final long newGen;
                        if (useUUIDs) {
                            newGen = -1L;
                            blobStoreIndexShardSnapshots = buildBlobStoreIndexShardSnapshots(blobs, shardContainer, oldRepositoryData.shardGenerations().getShardGen(indexId, finalShardId)).v1();
                        } else {
                            Tuple<BlobStoreIndexShardSnapshots, Long> tuple = buildBlobStoreIndexShardSnapshots(blobs, shardContainer);
                            newGen = tuple.v2() + 1;
                            blobStoreIndexShardSnapshots = tuple.v1();
                        }
                        allShardsListener.onResponse(deleteFromShardSnapshotMeta(survivingSnapshots, indexId, finalShardId, snapshotIds, shardContainer, blobs, blobStoreIndexShardSnapshots, newGen));
                    }

                    @Override
                    public void onFailure(Exception ex) {
                        logger.warn(() -> new ParameterizedMessage("{} failed to delete shard data for shard [{}][{}]", snapshotIds, indexId.getName(), finalShardId), ex);
                        // Just passing null here to count down the listener instead of failing it, the stale data left behind
                        // here will be retried in the next delete or repository cleanup
                        allShardsListener.onResponse(null);
                    }
                });
            }
        }, deleteIndexMetadataListener::onFailure);
    }
}